From fa954c7124c348fdba78cb1c05afeb5d18f1711d Mon Sep 17 00:00:00 2001 From: yiyione Date: Mon, 24 Aug 2020 10:15:09 +0800 Subject: [PATCH] Add parameter to list jobs (#57) * update swagger * add job list query to listJobs * fix tests * update * update * update * update --- .gitattributes | 1 + src/api/v2/clients/jobClient.ts | 8 +- src/api/v2/index.ts | 3 +- src/api/v2/models/job.ts | 24 + src/api/v2/swagger.yaml | 969 ++++++++++++++--------------- src/commands/jobCommands.ts | 2 +- tests/unit_tests/jobClient.spec.ts | 18 +- 7 files changed, 516 insertions(+), 509 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/src/api/v2/clients/jobClient.ts b/src/api/v2/clients/jobClient.ts index 7ceffa4..02eba6c 100644 --- a/src/api/v2/clients/jobClient.ts +++ b/src/api/v2/clients/jobClient.ts @@ -7,6 +7,8 @@ import { import { Util } from '@pai/commom/util'; import * as yaml from 'js-yaml'; +import { IJobListQeury } from '../models/job'; + import { OpenPAIBaseClient } from './baseClient'; /** @@ -37,15 +39,15 @@ export class JobClient extends OpenPAIBaseClient { /** * Get the list of jobs. - * @param username filter jobs with username. + * @param query filter jobs by username, vc, state and keyword. Set offset, limit, order and withTotalCount. */ - public async listJobs(username?: string): Promise { + public async listJobs(query?: IJobListQeury): Promise { const url: string = Util.fixUrl( `${this.cluster.rest_server_uri}/api/v2/jobs`, this.cluster.https ); return await this.httpClient.get( - url, undefined, undefined, { username } + url, undefined, undefined, query ); } diff --git a/src/api/v2/index.ts b/src/api/v2/index.ts index eea8f37..3ba3c98 100644 --- a/src/api/v2/index.ts +++ b/src/api/v2/index.ts @@ -20,7 +20,7 @@ import { StorageNodeV2 as StorageNode } from './clients/storageClient'; import { IAuthnInfo, ILoginInfo } from './models/authn'; import { IPAICluster, IPAIClusterInfo } from './models/cluster'; import { IGroup } from './models/group'; -import { IJobAttempt, IJobFrameworkInfo, IJobInfo, IJobSshInfo, IJobStatus } from './models/job'; +import { IJobAttempt, IJobFrameworkInfo, IJobInfo, IJobListQeury, IJobSshInfo, IJobStatus } from './models/job'; import { IPAIResponse } from './models/paiResponse'; import { IMountInfo, IStorageConfig, IStorageDetail, IStorageServer, IStorageSummary } from './models/storage'; import { IToken, ITokenList } from './models/token'; @@ -59,6 +59,7 @@ export { IStorageDetail, IMountInfo, IJobStatus, + IJobListQeury, StorageNode, IGroup, GroupClient, diff --git a/src/api/v2/models/job.ts b/src/api/v2/models/job.ts index 6500cb2..8552f0a 100644 --- a/src/api/v2/models/job.ts +++ b/src/api/v2/models/job.ts @@ -29,6 +29,30 @@ export interface IJobInfo { totalTaskRoleNumber: number; } +/** + * Query object for list job, filter jobs by username, vc, state and keyword. Set offset, limit, order and withTotalCount. + * e.g. + * { + * username: 'user1,user2', + * vc: 'vc1,vc2', + * state: 'RUNNING,WAITING,STOPPED', + * keyword: 'mnist', + * offset: 0, + * order: 'submissionTime,DESC', + * withTotalCount: false + * } + */ +export interface IJobListQeury { + username?: string; + vc?: string; + state?: string; + keyword?: string; + offset?: number; + limit?: number; + order?: string; // format ,, default value is "submissionTime,DESC" + withTotalCount?: boolean; +} + export interface IAppExitSpec { code: number; phrase: string; diff --git a/src/api/v2/swagger.yaml b/src/api/v2/swagger.yaml index e1be0b0..2d69754 100644 --- a/src/api/v2/swagger.yaml +++ b/src/api/v2/swagger.yaml @@ -6,13 +6,15 @@ info: Version 2.0.1: add more examples and fix typos Version 2.0.2: update job detail and job attempt schema Version 2.0.3: update parameters description of get storage list + Version 2.0.4: add default field in get storage list + Version 2.0.5: add more parameters to job list; add submissionTime license: name: MIT License - url: 'https://github.com/microsoft/pai/blob/master/LICENSE' - version: 2.0.3 + url: "https://github.com/microsoft/pai/blob/master/LICENSE" + version: 2.0.5 externalDocs: description: Find out more about OpenPAI - url: 'https://github.com/microsoft/pai' + url: "https://github.com/microsoft/pai" tags: - name: api description: API information @@ -43,7 +45,7 @@ paths: description: Get OpenPAI cluster info. operationId: getClusterInfo responses: - '200': + "200": description: Succeeded content: application/json: @@ -72,10 +74,10 @@ paths: - launcherType - authnMethod example: - name: 'PAI RESTful API' - version: 'v1.0.1' - launcherType: 'k8s' - authnMethod: 'basic' + name: "PAI RESTful API" + version: "v1.0.1" + launcherType: "k8s" + authnMethod: "basic" /api/v2/tokens: get: tags: @@ -86,7 +88,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -102,10 +104,10 @@ paths: - tokens example: tokens: - - 'JWT Token Example' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '/api/v2/tokens/{token}': + - "JWT Token Example" + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "/api/v2/tokens/{token}": delete: tags: - token @@ -115,20 +117,20 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/token' + - $ref: "#/components/parameters/token" responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: revoke successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" /api/v2/tokens/application: post: tags: @@ -142,7 +144,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -154,12 +156,12 @@ paths: description: your access token application: type: boolean - description: 'true' + description: "true" example: - token: 'JWT Token Example' + token: "JWT Token Example" application: true - '401': - $ref: '#/components/responses/UnauthorizedUserError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" /api/v2/authn/oidc/login: get: tags: @@ -168,7 +170,7 @@ paths: description: After call this API in web browser, restserver will redirect your page to Azure AD for authentication. Only be used in webportal. operationId: oidcLogin responses: - '302': + "302": description: Redirect /api/v2/authn/oidc/logout: get: @@ -178,7 +180,7 @@ paths: description: After call this API in web browser, restserver will redirect your page to Azure AD for logout. operationId: oidcLogout responses: - '302': + "302": description: Redirect /api/v2/authn/basic/login: post: @@ -210,7 +212,7 @@ paths: - password required: true responses: - '200': + "200": description: Succeeded content: application/json: @@ -231,17 +233,17 @@ paths: - user - admin example: - token: 'JWT Token Example' - user: 'user' + token: "JWT Token Example" + user: "user" admin: true - '400': + "400": description: NoUserError or IncorrectPasswordError content: application/json: schema: - $ref: '#/components/schemas/Response' - '404': - $ref: '#/components/responses/NoUserError' + $ref: "#/components/schemas/Response" + "404": + $ref: "#/components/responses/NoUserError" /api/v2/authn/basic/logout: delete: tags: @@ -252,24 +254,24 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: Logout successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" /api/v2/users: post: tags: - user summary: Create a user in the system. - description: 'Create a user in the system by admin, basic authentication mode only.' + description: "Create a user in the system by admin, basic authentication mode only." operationId: createUser security: - bearerAuth: [] @@ -303,32 +305,32 @@ paths: - password required: true responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: User is created successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '409': - $ref: '#/components/responses/ConflictUserError' - '500': - $ref: '#/components/responses/UnknownError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "409": + $ref: "#/components/responses/ConflictUserError" + "500": + $ref: "#/components/responses/UnknownError" get: tags: - user summary: Get all users in the system. - description: 'Get all users in the system by admin.' + description: "Get all users in the system by admin." operationId: getAllUser security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -336,16 +338,16 @@ paths: description: Array with all users in the system. type: array items: - $ref: '#/components/schemas/UserInfo' + $ref: "#/components/schemas/UserInfo" example: - username: username admin: true virtualCluster: [] storageConfig: [] - email: 'email@test.com' + email: "email@test.com" extension: {} - '500': - $ref: '#/components/responses/UnknownError' + "500": + $ref: "#/components/responses/UnknownError" put: tags: - user @@ -413,22 +415,22 @@ paths: type: boolean default: false responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'update group {username} successfully.' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' + message: "update group {username} successfully." + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" /api/v2/users/me: put: tags: @@ -477,25 +479,25 @@ paths: type: boolean default: false responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'update group {username} successfully.' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/users/{user}': + message: "update group {username} successfully." + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/users/{user}": parameters: - - $ref: '#/components/parameters/user' + - $ref: "#/components/parameters/user" get: tags: - user @@ -507,25 +509,25 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/UserInfo' + $ref: "#/components/schemas/UserInfo" example: username: username admin: true virtualCluster: [] storageConfig: [] - email: 'email@test.com' + email: "email@test.com" extension: {} - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" delete: tags: - user @@ -538,25 +540,25 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: user is removed successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/users/{user}/group/': + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/users/{user}/group/": parameters: - - $ref: '#/components/parameters/user' + - $ref: "#/components/parameters/user" put: tags: - user @@ -579,22 +581,22 @@ paths: type: string default: the group will be added into the user's grouplist. responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'User {username} is added into group {groupname}' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' + message: "User {username} is added into group {groupname}" + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" delete: tags: - user @@ -617,25 +619,25 @@ paths: type: string description: the group will be removed from the user's grouplist. responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'User {username} is removed from group {groupname}' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/users/{user}/grouplist/': + message: "User {username} is removed from group {groupname}" + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/users/{user}/grouplist/": parameters: - - $ref: '#/components/parameters/user' + - $ref: "#/components/parameters/user" put: tags: - user @@ -660,22 +662,22 @@ paths: items: type: string responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'update user grouplist successfully.' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoUserError' - '500': - $ref: '#/components/responses/UnknownError' + message: "update user grouplist successfully." + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoUserError" + "500": + $ref: "#/components/responses/UnknownError" /api/v2/groups: get: tags: @@ -686,7 +688,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -712,8 +714,8 @@ paths: description: description externalName: externalName extension: {} - '500': - $ref: '#/components/responses/UnknownError' + "500": + $ref: "#/components/responses/UnknownError" post: tags: - group @@ -746,20 +748,20 @@ paths: - groupname required: true responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: group is created successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '500': - $ref: '#/components/responses/UnknownError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "500": + $ref: "#/components/responses/UnknownError" put: tags: - group @@ -798,25 +800,25 @@ paths: default: false required: true responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'update group {groupname} successfully.' - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '404': - $ref: '#/components/responses/NoGroupError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/groups/{group}': + message: "update group {groupname} successfully." + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "404": + $ref: "#/components/responses/NoGroupError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/groups/{group}": parameters: - - $ref: '#/components/parameters/group' + - $ref: "#/components/parameters/group" get: tags: - group @@ -827,7 +829,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -855,12 +857,12 @@ paths: description: description externalName: externalName extension: {} - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '404': - $ref: '#/components/responses/NoGroupError' - '500': - $ref: '#/components/responses/UnknownError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "404": + $ref: "#/components/responses/NoGroupError" + "500": + $ref: "#/components/responses/UnknownError" delete: tags: - group @@ -872,23 +874,23 @@ paths: security: - bearerAuth: [] responses: - '201': + "201": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: group is removed successfully - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/groups/{group}/userlist': + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/groups/{group}/userlist": parameters: - - $ref: '#/components/parameters/group' + - $ref: "#/components/parameters/group" get: tags: - group @@ -896,7 +898,7 @@ paths: description: Get the user array of a group in the system. operationId: getGroupMembers responses: - '200': + "200": description: Succeeded content: application/json: @@ -915,10 +917,10 @@ paths: example: - username: username clusterAdmin: false - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '500': - $ref: '#/components/responses/UnknownError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "500": + $ref: "#/components/responses/UnknownError" /api/v2/virtual-clusters: get: tags: @@ -929,17 +931,17 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: schema: type: object additionalProperties: - $ref: '#/components/schemas/VirtualCluster' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/virtual-clusters/{vc}': + $ref: "#/components/schemas/VirtualCluster" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/virtual-clusters/{vc}": get: tags: - virtual cluster @@ -949,14 +951,14 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/vc' + - $ref: "#/components/parameters/vc" responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/VirtualCluster' + $ref: "#/components/schemas/VirtualCluster" example: capacity: 70 usedCapacity: 30 @@ -980,10 +982,10 @@ paths: vCores: 2 GPUs: 2 maxCapacity: 70 - '404': - $ref: '#/components/responses/NoVirtualClusterError' - '500': - $ref: '#/components/responses/UnknownError' + "404": + $ref: "#/components/responses/NoVirtualClusterError" + "500": + $ref: "#/components/responses/UnknownError" /api/v2/storages: get: tags: @@ -1000,20 +1002,21 @@ paths: schema: type: boolean responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/StorageSummary' + $ref: "#/components/schemas/StorageSummary" example: storages: - name: name share: true volumeName: volumeName - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/storages/{storage}': + default: true + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/storages/{storage}": get: tags: - storage @@ -1023,37 +1026,38 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/storage' + - $ref: "#/components/parameters/storage" responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/StorageDetail' + $ref: "#/components/schemas/StorageDetail" example: name: nfs-storage-name share: true volumeName: nfs-volume-name + default: true type: nfs data: server: path: "/data" - '403': + "403": description: ForbiddenUserError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: ForbiddenUserError: value: code: ForbiddenUserError - message: 'User {user} is not allowed to access {storage}.' - '404': - $ref: '#/components/responses/NoStorageError' - '500': - $ref: '#/components/responses/UnknownError' + message: "User {user} is not allowed to access {storage}." + "404": + $ref: "#/components/responses/NoStorageError" + "500": + $ref: "#/components/responses/UnknownError" /api/v2/jobs: post: tags: @@ -1068,25 +1072,25 @@ paths: content: text/yaml: schema: - $ref: '#/components/schemas/JobProtocol' + $ref: "#/components/schemas/JobProtocol" required: true responses: - '202': + "202": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: - message: 'update job {job} successfully' - '400': - $ref: '#/components/responses/NoVirtualClusterError' - '403': - $ref: '#/components/responses/ForbiddenUserError' - '409': - $ref: '#/components/responses/ConflictJobError' - '500': - $ref: '#/components/responses/UnknownError' + message: "update job {job} successfully" + "400": + $ref: "#/components/responses/NoVirtualClusterError" + "403": + $ref: "#/components/responses/ForbiddenUserError" + "409": + $ref: "#/components/responses/ConflictJobError" + "500": + $ref: "#/components/responses/UnknownError" get: tags: - job @@ -1101,28 +1105,66 @@ paths: description: filter jobs with username schema: type: string + - name: vc + in: query + description: filter jobs with virtual cluster name + schema: + type: string + - name: state + in: query + description: filter jobs with state + schema: + type: string + - name: keyword + in: query + description: filter jobs with keyword, we search keyword in user name, job name, and virtual cluster name + schema: + type: string + - name: offset + in: query + description: list job offset + schema: + type: number + - name: limit + in: query + description: list job limit. It has a default number 5000, and its max number is 50000. + schema: + type: number + - name: order + in: query + description: 'order of job list. It follows the format ,, default value is "submissionTime,DESC". Available fields include: jobName, submissionTime, username, vc, retries, totalTaskNumber, totalGpuNumber, state' + schema: + type: string + - name: withTotalCount + in: query + description: 'if withTotalCount is "true", the result will contain a "totalCount" and a "data" field, instead of a job list.' + schema: + type: string responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/JobSummary' + oneOf: + - $ref: "#/components/schemas/JobSummary" + - $ref: "#/components/schemas/JobSummaryWithTotalCount" example: - - protocolVersion: '2' + - protocolVersion: "2" name: job name username: user name state: SUCCEEDED subState: Completed executionType: STOP retries: 0 + submissionTime: 0 createdTime: 0 completedTime: 0 appExitCode: 0 virtualCluster: unknown - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/jobs/{user}~{job}': + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/jobs/{user}~{job}": get: tags: - job @@ -1132,15 +1174,15 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/JobDetail' + $ref: "#/components/schemas/JobDetail" example: name: job name jobStatus: @@ -1149,6 +1191,7 @@ paths: subState: Completed executionType: STOP retries: 0 + submissionTime: 0 createdTime: 0 completedTime: 0 appId: id @@ -1163,11 +1206,11 @@ paths: taskStatuses: - taskIndex: 0 taskState: SUCCEEDED - '404': - $ref: '#/components/responses/NoJobError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/jobs/{user}~{job}/config': + "404": + $ref: "#/components/responses/NoJobError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/jobs/{user}~{job}/config": get: tags: - job @@ -1177,47 +1220,48 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" responses: - '200': + "200": description: Succeeded content: text/yaml: schema: - $ref: '#/components/schemas/JobProtocol' + $ref: "#/components/schemas/JobProtocol" example: - protocolVersion: '2' - name: + protocolVersion: 2 + name: jobconfig type: job + version: '1.0' prerequisites: - - name: + - name: image type: dockerimage - uri: image_url + uri : uri taskRoles: - train: + taskRole: instances: 1 - dockerImage: + dockerImage: image resourcePerInstance: cpu: 1 - memoryMB: 16384 + memoryMB: 1024 gpu: 1 commands: - - python - '404': + - commands + "404": description: NoJobError or NoJobConfigError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoJobError: - $ref: '#/components/responses/NoJobError/content/application~1json/examples/NoJobError' + $ref: "#/components/responses/NoJobError/content/application~1json/examples/NoJobError" NoJobConfigError: - $ref: '#/components/responses/NoJobConfigError/content/application~1json/examples/NoJobConfigError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/jobs/{user}~{job}/executionType': + $ref: "#/components/responses/NoJobConfigError/content/application~1json/examples/NoJobConfigError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/jobs/{user}~{job}/executionType": put: tags: - job @@ -1227,8 +1271,8 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" requestBody: description: Execution type content: @@ -1246,19 +1290,19 @@ paths: - value required: true responses: - '202': + "202": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" example: message: execute job {job} successfully - '404': - $ref: '#/components/responses/NoJobError' - '500': - $ref: '#/components/responses/UnknownError' - '/api/v2/jobs/{user}~{job}/job-attempts/healthz': + "404": + $ref: "#/components/responses/NoJobError" + "500": + $ref: "#/components/responses/UnknownError" + "/api/v2/jobs/{user}~{job}/job-attempts/healthz": get: tags: - job history @@ -1268,14 +1312,14 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" responses: - '200': + "200": description: OK - '501': + "501": description: Not healthy - '/api/v2/jobs/{user}~{job}/job-attempts': + "/api/v2/jobs/{user}~{job}/job-attempts": get: tags: - job history @@ -1285,17 +1329,17 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" responses: - '200': + "200": description: Succeeded content: application/json: schema: type: array items: - $ref: '#/components/schemas/JobAttempt' + $ref: "#/components/schemas/JobAttempt" description: job attempts example: - jobName: jobName @@ -1316,7 +1360,7 @@ paths: diagnosticsSummary: Pod succeeded runtime: launcher: Pod succeeded - appExitTriggerMessage: 'All Tasks are completed' + appExitTriggerMessage: "All Tasks are completed" appExitTriggerTaskRoleName: taskrole appExitTriggerTaskIndex: 0 appExitSpec: @@ -1349,11 +1393,11 @@ paths: containerIp: containerExitCode: 0 isLatest: true - '404': - $ref: '#/components/responses/NoJobError' - '501': - $ref: '#/components/responses/UnknownError' - '/api/v2/jobs/{user}~{job}/job-attempts/{attemptIndex}': + "404": + $ref: "#/components/responses/NoJobError" + "501": + $ref: "#/components/responses/UnknownError" + "/api/v2/jobs/{user}~{job}/job-attempts/{attemptIndex}": get: tags: - job history @@ -1363,16 +1407,16 @@ paths: security: - bearerAuth: [] parameters: - - $ref: '#/components/parameters/user' - - $ref: '#/components/parameters/job' - - $ref: '#/components/parameters/attemptIndex' + - $ref: "#/components/parameters/user" + - $ref: "#/components/parameters/job" + - $ref: "#/components/parameters/attemptIndex" responses: - '200': + "200": description: Succeeded content: application/json: schema: - $ref: '#/components/schemas/JobAttempt' + $ref: "#/components/schemas/JobAttempt" example: jobName: jobName frameworkName: frameworkName @@ -1392,7 +1436,7 @@ paths: diagnosticsSummary: Pod succeeded runtime: launcher: Pod succeeded - appExitTriggerMessage: 'All Tasks are completed' + appExitTriggerMessage: "All Tasks are completed" appExitTriggerTaskRoleName: taskrole appExitTriggerTaskIndex: 0 appExitSpec: @@ -1425,10 +1469,10 @@ paths: containerIp: containerExitCode: 0 isLatest: true - '404': - $ref: '#/components/responses/NoJobError' - '501': - $ref: '#/components/responses/UnknownError' + "404": + $ref: "#/components/responses/NoJobError" + "501": + $ref: "#/components/responses/UnknownError" /api/v2/kubernetes/nodes: get: tags: @@ -1439,7 +1483,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -1447,10 +1491,10 @@ paths: - Please refer to Kubernetes API doc - >- https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#list-node-v1-core - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" /api/v2/kubernetes/pods: get: tags: @@ -1467,7 +1511,7 @@ paths: security: - bearerAuth: [] responses: - '200': + "200": description: Succeeded content: application/json: @@ -1475,10 +1519,10 @@ paths: - Please refer to Kubernetes API doc - >- https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#list-all-namespaces-pod-v1-core - '401': - $ref: '#/components/responses/UnauthorizedUserError' - '403': - $ref: '#/components/responses/ForbiddenUserError' + "401": + $ref: "#/components/responses/UnauthorizedUserError" + "403": + $ref: "#/components/responses/ForbiddenUserError" components: parameters: token: @@ -1523,7 +1567,7 @@ components: framework: name: framework in: path - description: 'framework name defined by {user}~{job}' + description: "framework name defined by {user}~{job}" required: true schema: type: string @@ -1576,187 +1620,18 @@ components: required: - message JobProtocol: + $ref: "https://raw.githubusercontent.com/microsoft/openpai-protocol/master/schemas/v2/schema.yaml#/components/schemas/JobProtocol" + JobSummaryWithTotalCount: type: object - description: openpai job protocol properties: - protocolVersion: - oneOf: - - type: number - - type: string - description: Protocol version, current version is 2. - name: - type: string - maxLength: 255 - description: String in ^[a-zA-Z0-9_-]+$ format, no longer than 255 characters. - type: - type: string - description: Component type, should be "job" here. - version: - type: string - description: Component version, default is latest. - contributor: - type: string - description: - type: string - prerequisites: - type: array - items: - type: object - properties: - name: - type: string - type: - type: string - enum: - - data - - script - - dockerimage - - output - version: - type: string - contributor: - type: string - description: - type: string - auth: - type: object - properties: - username: - type: string - password: - type: string - registryuri: - type: string - uri: - oneOf: - - type: string - - type: array - items: - type: string - parameters: - type: object - additionalProperties: true - secrets: - type: object - additionalProperties: true - jobRetryCount: - type: integer - description: Default is 0. - taskRoles: - type: object - additionalProperties: - type: object - properties: - instances: - type: integer - description: Default is 1, instances of a taskRole, no less than 1. - completion: - type: object - description: Completion poclicy for the job. - properties: - minFailedInstances: - type: integer - minSucceededInstances: - type: integer - taskRetryCount: - type: integer - description: Default is 0. - dockerImage: - type: string - description: Should reference to a dockerimage defined in prerequisites. - data: - type: string - output: - type: string - script: - type: string - extraContainerOptions: - type: object - properties: - shmMB: - type: integer - description: Config the /dev/shm in a docker container. - infiniband: - type: boolean - description: Use InfiniBand devices or not in a docker container. - resourcePerInstance: - type: object - properties: - cpu: - type: integer - description: CPU number, unit is CPU vcore. - memoryMB: - type: integer - description: Memory number, unit is MB. - gpu: - type: integer - description: GPU number, unit is GPU card. - ports: - type: object - additionalProperties: - type: integer - description: Port number for the port label. - commands: - type: array - items: - type: string - required: - - dockerImage - - resourcePerInstance - - commands - deployments: - type: array - items: - type: object - properties: - name: - type: string - taskRoles: - type: object - additionalProperties: - type: object - properties: - preCommands: - type: array - items: - type: string - postCommands: - type: array - items: - type: string - defaults: - type: object - description: Default cluster specific settings. - properties: - virtualCluster: - type: string - deployment: - type: string - extras: - type: object - description: Extra field, save any information that plugin may use. - properties: - submitFrom: - type: string - hivedScheduler: - type: object - properties: - jobPriorityClass: - type: string - taskRoles: - type: object - additionalProperties: - type: object - properties: - skuType: - type: string - affinityGroupName: - type: string + totalCount: + type: number + description: total count of jobs with given filters + data: + $ref: "#/components/schemas/JobSummary" required: - - protocolVersion - - name - - type - - taskRoles + - totalCount + - data JobSummary: type: array description: job summary list @@ -1815,8 +1690,12 @@ components: type: integer nullable: true description: retry delay time + submissionTime: + type: integer + description: job submitted time, in number of milliseconds since the Unix Epoch. createdTime: type: integer + nullable: true description: job created time, in number of milliseconds since the Unix Epoch. completedTime: type: integer @@ -1912,8 +1791,14 @@ components: type: integer nullable: true description: retry delay time + submissionTime: + type: integer + description: >- + job submitted time, in number of milliseconds since the Unix + Epoch. createdTime: type: integer + nullable: true description: >- job created time, in number of milliseconds since the Unix Epoch. @@ -1968,10 +1853,16 @@ components: type: string reaction: type: string + reason: + type: string repro: type: array items: type: string + solution: + type: array + items: + type: string appExitDiagnostics: type: string nullable: true @@ -2062,6 +1953,10 @@ components: type: string nullable: true description: ip of the task container + containerNodeName: + type: string + nullable: true + description: node name of task container containerPorts: type: object description: ports of the task container @@ -2076,6 +1971,72 @@ components: type: integer nullable: true description: exit code the task container + containerExitSpec: + type: object + nullable: true + description: container exit spec + properties: + code: + type: integer + phrase: + type: string + issuer: + type: string + causer: + type: string + type: + type: string + stage: + type: string + behavior: + type: string + reaction: + type: string + reason: + type: string + repro: + type: array + items: + type: string + solution: + type: array + items: + type: string + containerExitDiagnostics: + type: string + nullable: true + description: container exit diagnostics + retries: + type: integer + accountableRetries: + type: integer + createdTime: + type: integer + nullable: true + description: >- + task created time, in number of milliseconds since the Unix + Epoch. + completedTime: + type: integer + nullable: true + description: >- + task completion time, in number of milliseconds since the Unix + Epoch. + currentAttemptLaunchedTime: + type: integer + nullable: true + description: >- + the current attempt launched time, in number of milliseconds since the Unix + Epoch. + currentAttemptCompletedTime: + type: integer + nullable: true + description: >- + the current attempt completion time, in number of milliseconds since the Unix + Epoch. + hived: + type: object + nullable: true required: - taskRoleStatus - taskStatuses @@ -2181,10 +2142,13 @@ components: type: boolean volumeName: type: string + default: + type: boolean required: - name - share - volumeName + - default required: - storages StorageDetail: @@ -2294,6 +2258,7 @@ components: nullable: true jobStartedTime: type: integer + nullable: true attemptStartedTime: type: integer nullable: true @@ -2502,7 +2467,7 @@ components: content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: IncorrectPasswordError: value: @@ -2513,40 +2478,40 @@ components: content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoVirtualClusterError: value: code: NoVirtualClusterError - message: 'Virtual cluster {vc} is not found.' + message: "Virtual cluster {vc} is not found." NoGroupError: description: NoGroupError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoGroupError: value: code: NoGroupError - message: 'Group {groupname} is not found.' + message: "Group {groupname} is not found." NoStorageError: description: NoStorageError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoVirtualClusterError: value: code: NoStorageError - message: 'Storage {storage} is not found.' + message: "Storage {storage} is not found." UnauthorizedUserError: description: UnauthorizedUserError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: UnauthorizedUserError: value: @@ -2557,7 +2522,7 @@ components: content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: ForbiddenUserError: value: @@ -2572,7 +2537,7 @@ components: content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: ForbiddenKeyError: value: @@ -2583,73 +2548,73 @@ components: content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoUserError: value: code: NoUserError - message: 'User {user} is not found.' + message: "User {user} is not found." NoJobError: description: NoJobError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoJobError: value: code: NoJobError - message: 'Job {job} is not found.' + message: "Job {job} is not found." NoJobConfigError: description: NoJobConfigError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoJobConfigError: value: code: NoJobConfigError - message: 'Config of job {job} is not found.' + message: "Config of job {job} is not found." NoJobSshInfoError: description: NoJobSshInfoError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: NoJobSshInfoError: value: code: NoJobSshInfoError - message: 'SSH info of job {job} is not found.' + message: "SSH info of job {job} is not found." ConflictUserError: description: ConflictUserError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: ConflictUserError: value: code: ConflictUserError - message: 'User name {user} already exists.' + message: "User name {user} already exists." ConflictJobError: description: ConflictJobError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: ConflictJobError: value: code: ConflictJobError - message: 'Job name {job} already exists.' + message: "Job name {job} already exists." UnknownError: description: UnknownError content: application/json: schema: - $ref: '#/components/schemas/Response' + $ref: "#/components/schemas/Response" examples: UnknownError: value: diff --git a/src/commands/jobCommands.ts b/src/commands/jobCommands.ts index a95870b..d495b51 100644 --- a/src/commands/jobCommands.ts +++ b/src/commands/jobCommands.ts @@ -25,7 +25,7 @@ export function registerJobCommands(cli: CliEngine): void { if (a.all) { return client.job.listJobs(); } - return client.job.listJobs(a.user || client.config.username()); + return client.job.listJobs({ username: a.user || client.config.username() }); }, [ { diff --git a/tests/unit_tests/jobClient.spec.ts b/tests/unit_tests/jobClient.spec.ts index 994f3c7..7be8f19 100644 --- a/tests/unit_tests/jobClient.spec.ts +++ b/tests/unit_tests/jobClient.spec.ts @@ -37,7 +37,7 @@ describe('List jobs', () => { // tslint:disable-next-line:mocha-no-side-effect-code it('should return a list of jobs', async () => { const jobClient: JobClient = new JobClient(cluster); - const result: IJobInfo[] = await jobClient.listJobs(); + const result: any = await jobClient.listJobs(); expect(result).is.not.empty(); }).timeout(10000); }); @@ -50,11 +50,25 @@ describe('List jobs with query', () => { // tslint:disable-next-line:mocha-no-side-effect-code it('should return a list of jobs', async () => { const jobClient: JobClient = new JobClient(cluster); - const result: IJobInfo[] = await jobClient.listJobs('core'); + const result: any = await jobClient.listJobs({ username: 'core' }); expect(result).is.not.empty(); }).timeout(10000); }); +describe('List jobs with total count', () => { + const response: any = { totalCount: 1, data: testJobList }; + const queryString: string = 'username=core&withTotalCount=true'; + before(() => nock(`http://${testUri}`).get(`/api/v2/jobs?${queryString}`).reply(200, response)); + + // tslint:disable-next-line:mocha-no-side-effect-code + it('should return a list of jobs', async () => { + const jobClient: JobClient = new JobClient(cluster); + const result: any = await jobClient.listJobs({ username: 'core', withTotalCount: true }); + expect(result.totalCount).equal(1); + expect(result.data).is.not.empty(); + }).timeout(10000); +}); + describe('Get job status', () => { const response: IJobStatus = testJobStatus; const userName: string = 'core';