Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(controller): make ds build a job #2626

Merged
merged 2 commits into from
Aug 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions console/src/domain/dataset/schemas/dataset.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export interface IDatasetTaskBuildSchema {
id: string
datasetId: string
projectId: string
taskId: string
datasetName: string
status: TaskBuildStatusType
type: string
Expand All @@ -62,6 +63,7 @@ export enum TaskBuildStatusType {
CREATED = 'CREATED',
UPLOADING = 'UPLOADING',
BUILDING = 'BUILDING',
RUNNING = 'RUNNING',
SUCCESS = 'SUCCESS',
FAILED = 'FAILED',
}
9 changes: 7 additions & 2 deletions console/src/domain/dataset/services/dataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,12 @@ export async function fetchDatasetBuildList(
return resp.data
}

export async function fetchDatasetTaskOfflineLogFiles(datasetName: string, taskId: string): Promise<any> {
const resp = await axios.get<string[]>(`/api/v1/log/offline/dataset/${datasetName}/build/${taskId}`)
export async function fetchDatasetTaskOfflineLogFiles(taskId: string): Promise<any> {
const resp = await axios.get<string[]>(`/api/v1/log/offline/${taskId}`)
return resp.data
}

export async function fetchTaskOfflineFileLog(taskId: string, fileId: string): Promise<any> {
const resp = await axios.get<string>(`/api/v1/log/offline/${taskId}/${fileId}`)
return resp.data
}
23 changes: 16 additions & 7 deletions console/src/pages/Dataset/DatasetBuildListCard.tsx
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import React, { useCallback, useMemo, useState } from 'react'
import _ from 'lodash'
import useTranslation from '@/hooks/useTranslation'
import Card from '@/components/Card'
import { getToken } from '@/api'
import { toaster } from 'baseui/toast'
import { BusyPlaceholder } from '@starwhale/ui'
import DatasetTaskBuildList from './DatasetBuildList'
import { fetchDatasetTaskOfflineLogFiles } from '@/domain/dataset/services/dataset'
import { fetchDatasetTaskOfflineLogFiles, fetchTaskOfflineFileLog } from '@/domain/dataset/services/dataset'
import { IDatasetTaskBuildSchema, TaskBuildStatusType } from '@/domain/dataset/schemas/dataset'

const ComplexToolbarLogViewer = React.lazy(() => import('@/components/LogViewer/LogViewer'))
Expand All @@ -28,11 +29,19 @@ export default function DatasetBuildListCard() {
const files: Record<string, string> = {}
const key = [task?.datasetName, task?.id].join('@')

if ([TaskBuildStatusType.BUILDING].includes(task.status)) {
if ([TaskBuildStatusType.RUNNING].includes(task.status)) {
files[key] = 'ws'
} else {
const data = await fetchDatasetTaskOfflineLogFiles(task?.datasetName, task?.id)
files[key] = data ?? ''
const data = await fetchDatasetTaskOfflineLogFiles(task?.taskId)
// files[key] = data ?? ''
if (!_.isEmpty(data)) {
await Promise.all(
data.map(async (v: string) => {
const content = await fetchTaskOfflineFileLog(task?.taskId, v)
files[key] = content ?? ''
})
)
}
}

if (Object.keys(files).length === 0) {
Expand All @@ -47,9 +56,9 @@ export default function DatasetBuildListCard() {
}, [])

const currentOnlineLogUrl = useMemo(() => {
return `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${
window.location.host
}/api/v1/log/online/dataset/${currentTask?.datasetName}/build/${currentTask?.id}?Authorization=${getToken()}`
return `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}/api/v1/log/online/${
currentTask?.taskId
}?Authorization=${getToken()}`
}, [currentTask])

const sources = React.useMemo(() => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import ai.starwhale.mlops.api.protocol.dataset.dataloader.DataIndexDesc;
import ai.starwhale.mlops.api.protocol.dataset.upload.DatasetUploadRequest;
import ai.starwhale.mlops.api.protocol.upload.UploadResult;
import ai.starwhale.mlops.domain.dataset.build.BuildStatus;
import com.github.pagehelper.PageInfo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand Down Expand Up @@ -460,8 +459,6 @@ ResponseEntity<ResponseMessage<PageInfo<BuildRecordVo>>> listBuildRecords(
@Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
@PathVariable(name = "projectUrl")
String projectUrl,
@RequestParam(value = "status", required = false)
BuildStatus status,
@Valid @RequestParam(value = "pageNum", required = false, defaultValue = "1")
Integer pageNum,
@Valid @RequestParam(value = "pageSize", required = false, defaultValue = "10")
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import ai.starwhale.mlops.domain.dataset.DatasetService;
import ai.starwhale.mlops.domain.dataset.bo.DatasetQuery;
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersionQuery;
import ai.starwhale.mlops.domain.dataset.build.BuildStatus;
import ai.starwhale.mlops.domain.dataset.build.bo.CreateBuildRecordRequest;
import ai.starwhale.mlops.domain.dataset.dataloader.DataReadRequest;
import ai.starwhale.mlops.domain.dataset.dataloader.ReadMode;
Expand Down Expand Up @@ -456,7 +455,6 @@ public ResponseEntity<ResponseMessage<String>> buildDataset(
DatasetBuildRequest datasetBuildRequest
) {
datasetService.build(CreateBuildRecordRequest.builder()
.datasetId(datasetBuildRequest.getDatasetId())
.datasetName(datasetName)
.shared(datasetBuildRequest.getShared())
.projectUrl(projectUrl)
Expand All @@ -469,12 +467,11 @@ public ResponseEntity<ResponseMessage<String>> buildDataset(
@Override
public ResponseEntity<ResponseMessage<PageInfo<BuildRecordVo>>> listBuildRecords(
String projectUrl,
BuildStatus status,
Integer pageNum,
Integer pageSize
) {
return ResponseEntity.ok(Code.success.asResponse(
datasetService.listBuildRecords(projectUrl, status, new PageParams(pageNum, pageSize))));
datasetService.listBuildRecords(projectUrl, new PageParams(pageNum, pageSize))));
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import ai.starwhale.mlops.configuration.FeaturesProperties;
import ai.starwhale.mlops.domain.dag.DagQuerier;
import ai.starwhale.mlops.domain.dag.bo.Graph;
import ai.starwhale.mlops.domain.job.JobService;
import ai.starwhale.mlops.domain.job.JobServiceForWeb;
import ai.starwhale.mlops.domain.job.ModelServingService;
import ai.starwhale.mlops.domain.job.RuntimeSuggestionService;
import ai.starwhale.mlops.domain.task.TaskService;
Expand All @@ -56,7 +56,7 @@
@RequestMapping("${sw.controller.api-prefix}")
public class JobController implements JobApi {

private final JobService jobService;
private final JobServiceForWeb jobServiceForWeb;
private final TaskService taskService;
private final ModelServingService modelServingService;
private final RuntimeSuggestionService runtimeSuggestionService;
Expand All @@ -66,28 +66,28 @@ public class JobController implements JobApi {
private final FeaturesProperties featuresProperties;

public JobController(
JobService jobService,
JobServiceForWeb jobServiceForWeb,
TaskService taskService,
ModelServingService modelServingService,
RuntimeSuggestionService runtimeSuggestionService,
IdConverter idConvertor,
DagQuerier dagQuerier,
FeaturesProperties featuresProperties
) {
this.jobService = jobService;
this.jobServiceForWeb = jobServiceForWeb;
this.taskService = taskService;
this.modelServingService = modelServingService;
this.runtimeSuggestionService = runtimeSuggestionService;
this.idConvertor = idConvertor;
this.dagQuerier = dagQuerier;
this.featuresProperties = featuresProperties;
var actions = InvokerManager.<String, String>create()
.addInvoker("cancel", jobService::cancelJob);
.addInvoker("cancel", jobServiceForWeb::cancelJob);
if (featuresProperties.isJobPauseEnabled()) {
actions.addInvoker("pause", jobService::pauseJob);
actions.addInvoker("pause", jobServiceForWeb::pauseJob);
}
if (featuresProperties.isJobResumeEnabled()) {
actions.addInvoker("resume", jobService::resumeJob);
actions.addInvoker("resume", jobServiceForWeb::resumeJob);
}
this.jobActions = actions.unmodifiable();
}
Expand All @@ -100,7 +100,7 @@ public ResponseEntity<ResponseMessage<PageInfo<JobVo>>> listJobs(
Integer pageSize
) {

PageInfo<JobVo> jobVos = jobService.listJobs(projectUrl, idConvertor.revert(modelId),
PageInfo<JobVo> jobVos = jobServiceForWeb.listJobs(projectUrl, idConvertor.revert(modelId),
PageParams.builder()
.pageNum(pageNum)
.pageSize(pageSize)
Expand All @@ -110,7 +110,7 @@ public ResponseEntity<ResponseMessage<PageInfo<JobVo>>> listJobs(

@Override
public ResponseEntity<ResponseMessage<JobVo>> findJob(String projectUrl, String jobUrl) {
JobVo job = jobService.findJob(projectUrl, jobUrl);
JobVo job = jobServiceForWeb.findJob(projectUrl, jobUrl);
return ResponseEntity.ok(Code.success.asResponse(job));
}

Expand Down Expand Up @@ -144,7 +144,7 @@ public ResponseEntity<ResponseMessage<String>> createJob(
throw new StarwhaleApiException(new SwValidationException(ValidSubject.JOB, "dev mode is not enabled"),
HttpStatus.BAD_REQUEST);
}
Long jobId = jobService.createJob(projectUrl,
Long jobId = jobServiceForWeb.createJob(projectUrl,
jobRequest.getModelVersionUrl(),
jobRequest.getDatasetVersionUrls(),
jobRequest.getRuntimeVersionUrl(),
Expand Down Expand Up @@ -179,7 +179,7 @@ public ResponseEntity<ResponseMessage<String>> action(

@Override
public ResponseEntity<ResponseMessage<Object>> getJobResult(String projectUrl, String jobUrl) {
Object jobResult = jobService.getJobResult(projectUrl, jobUrl);
Object jobResult = jobServiceForWeb.getJobResult(projectUrl, jobUrl);
return ResponseEntity.ok(Code.success.asResponse(jobResult));
}

Expand All @@ -189,7 +189,7 @@ public ResponseEntity<ResponseMessage<String>> modifyJobComment(
String jobUrl,
JobModifyRequest jobModifyRequest
) {
Boolean res = jobService.updateJobComment(projectUrl, jobUrl, jobModifyRequest.getComment());
Boolean res = jobServiceForWeb.updateJobComment(projectUrl, jobUrl, jobModifyRequest.getComment());

if (!res) {
throw new StarwhaleApiException(new SwProcessException(ErrorType.DB, "Update job comment failed."),
Expand All @@ -204,7 +204,7 @@ public ResponseEntity<ResponseMessage<String>> modifyJobPinStatus(
String jobUrl,
JobModifyPinRequest jobRequest
) {
Boolean res = jobService.updateJobPinStatus(projectUrl, jobUrl, jobRequest.isPinned());
Boolean res = jobServiceForWeb.updateJobPinStatus(projectUrl, jobUrl, jobRequest.isPinned());

if (!res) {
throw new StarwhaleApiException(new SwProcessException(ErrorType.DB, "Update job pin status failed."),
Expand All @@ -220,7 +220,7 @@ public ResponseEntity<ResponseMessage<Graph>> getJobDag(String projectUrl, Strin

@Override
public ResponseEntity<ResponseMessage<String>> removeJob(String projectUrl, String jobUrl) {
Boolean res = jobService.removeJob(projectUrl, jobUrl);
Boolean res = jobServiceForWeb.removeJob(projectUrl, jobUrl);
if (!res) {
throw new StarwhaleApiException(new SwProcessException(ErrorType.DB, "Remove job failed."),
HttpStatus.INTERNAL_SERVER_ERROR);
Expand All @@ -230,7 +230,7 @@ public ResponseEntity<ResponseMessage<String>> removeJob(String projectUrl, Stri

@Override
public ResponseEntity<ResponseMessage<String>> recoverJob(String projectUrl, String jobUrl) {
Boolean res = jobService.recoverJob(projectUrl, jobUrl);
Boolean res = jobServiceForWeb.recoverJob(projectUrl, jobUrl);
if (!res) {
throw new StarwhaleApiException(new SwProcessException(ErrorType.DB, "Recover job failed."),
HttpStatus.INTERNAL_SERVER_ERROR);
Expand Down Expand Up @@ -288,7 +288,7 @@ public ResponseEntity<ResponseMessage<ExecResponse>> exec(
String taskId,
ExecRequest execRequest
) {
var resp = jobService.exec(projectUrl, jobUrl, taskId, execRequest);
var resp = jobServiceForWeb.exec(projectUrl, jobUrl, taskId, execRequest);
return ResponseEntity.ok(Code.success.asResponse(resp));
}
}
Loading
Loading