refactor(controller): resource pool implementation change;update doc (#…

…1420) * refactor resource pool implementation;update doc * fix ut error
star-whale · Oct 25, 2022 · 0ff8d96 · 0ff8d96
1 parent 53bf274
commit 0ff8d96
Show file tree

Hide file tree

Showing 39 changed files with 145 additions and 780 deletions.
diff --git a/docs/docs/community/contribute.md b/docs/docs/community/contribute.md
@@ -185,11 +185,11 @@ mvn clean test
   - Minio
 
     ```bash
-    docker run --name minio -d
-    -p 9000:9000  --publish 9001:9001
-    -e MINIO_DEFAULT_BUCKETS='starwhale'
-    -e MINIO_ROOT_USER="minioadmin"
-    -e MINIO_ROOT_PASSWORD="minioadmin"
+    docker run --name minio -d \
+    -p 9000:9000  --publish 9001:9001 \
+    -e MINIO_DEFAULT_BUCKETS='starwhale' \
+    -e MINIO_ROOT_USER="minioadmin" \
+    -e MINIO_ROOT_PASSWORD="minioadmin" \
     bitnami/minio:latest
     ```
 

diff --git a/scripts/e2e_test/docker/entrypoint.sh b/scripts/e2e_test/docker/entrypoint.sh
@@ -34,7 +34,7 @@ git config --global --add safe.directory /starwhale
 git config --global user.email "renyanda@starwhale.ai"
 cd /starwhale/scripts/e2e_test
 if [[ -z "$PUBLISH" ]] ; then
-  if source start_test.sh ;then
+  if bash start_test.sh ;then
     send_feishu "e2e SUCCESS ns:$SWNS name:$SWNAME"
   else
     send_feishu "e2e FAIL ns:$SWNS name:$SWNAME"

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/SystemApi.java b/server/controller/src/main/java/ai/starwhale/mlops/api/SystemApi.java
@@ -17,9 +17,9 @@
 package ai.starwhale.mlops.api;
 
 import ai.starwhale.mlops.api.protocol.ResponseMessage;
-import ai.starwhale.mlops.api.protocol.system.ResourcePoolVo;
 import ai.starwhale.mlops.api.protocol.system.SystemVersionVo;
 import ai.starwhale.mlops.api.protocol.system.UpgradeProgressVo;
+import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.enums.ParameterIn;
@@ -53,7 +53,7 @@ public interface SystemApi {
                                     schema = @Schema(implementation = List.class)))
             })
     @GetMapping(value = "/system/resourcePool")
-    ResponseEntity<ResponseMessage<List<ResourcePoolVo>>> listResourcePools();
+    ResponseEntity<ResponseMessage<List<ResourcePool>>> listResourcePools();
 
     @Operation(summary = "Upgrade system version or cancel upgrade")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "ok")})

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/SystemController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/SystemController.java
@@ -18,12 +18,12 @@
 
 import ai.starwhale.mlops.api.protocol.Code;
 import ai.starwhale.mlops.api.protocol.ResponseMessage;
-import ai.starwhale.mlops.api.protocol.system.ResourcePoolVo;
 import ai.starwhale.mlops.api.protocol.system.SystemVersionVo;
 import ai.starwhale.mlops.api.protocol.system.UpgradeProgressVo;
 import ai.starwhale.mlops.api.protocol.system.UpgradeProgressVo.PhaseEnum;
 import ai.starwhale.mlops.domain.system.SystemService;
 import ai.starwhale.mlops.domain.system.SystemSettingService;
+import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import java.util.List;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -44,7 +44,7 @@ public SystemController(SystemService systemService,
     }
 
     @Override
-    public ResponseEntity<ResponseMessage<List<ResourcePoolVo>>> listResourcePools() {
+    public ResponseEntity<ResponseMessage<List<ResourcePool>>> listResourcePools() {
         return ResponseEntity.ok(Code.success.asResponse(systemService.listResourcePools()));
     }
 

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/task/TaskVo.java b/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/task/TaskVo.java
@@ -49,6 +49,6 @@ public class TaskVo implements Serializable {
     private String resourcePool;
 
     public static TaskVo empty() {
-        return new TaskVo("", "", -1L, TaskStatus.CREATED, ResourcePool.DEFAULT);
+        return new TaskVo("", "", -1L, TaskStatus.CREATED, ResourcePool.DEFAULT_NAME);
     }
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java
@@ -37,7 +37,6 @@
 import ai.starwhale.mlops.domain.storage.StoragePathCoordinator;
 import ai.starwhale.mlops.domain.swds.SwdsManager;
 import ai.starwhale.mlops.domain.swmp.SwmpManager;
-import ai.starwhale.mlops.domain.system.resourcepool.ResourcePoolManager;
 import ai.starwhale.mlops.domain.task.bo.Task;
 import ai.starwhale.mlops.domain.task.mapper.TaskMapper;
 import ai.starwhale.mlops.domain.task.status.TaskStatus;
@@ -89,12 +88,11 @@ public class JobService {
     private final SwmpManager swmpManager;
     private final SwdsManager swdsManager;
     private final RuntimeManager runtimeManager;
-    private final ResourcePoolManager resourcePoolManager;
     private final JobUpdateHelper jobUpdateHelper;
 
     public JobService(JobBoConverter jobBoConverter, JobMapper jobMapper, JobSwdsVersionMapper jobSwdsVersionMapper,
             TaskMapper taskMapper, JobConvertor jobConvertor, RuntimeManager runtimeManager,
-            JobSpliterator jobSpliterator, ResourcePoolManager resourcePoolManager, HotJobHolder hotJobHolder,
+            JobSpliterator jobSpliterator,  HotJobHolder hotJobHolder,
             ProjectManager projectManager, JobManager jobManager, JobLoader jobLoader, SwmpManager swmpManager,
             ResultQuerier resultQuerier, SwdsManager swdsManager, StoragePathCoordinator storagePathCoordinator,
             UserService userService, JobUpdateHelper jobUpdateHelper) {
@@ -105,7 +103,6 @@ public JobService(JobBoConverter jobBoConverter, JobMapper jobMapper, JobSwdsVer
         this.jobConvertor = jobConvertor;
         this.runtimeManager = runtimeManager;
         this.jobSpliterator = jobSpliterator;
-        this.resourcePoolManager = resourcePoolManager;
         this.hotJobHolder = hotJobHolder;
         this.projectManager = projectManager;
         this.jobManager = jobManager;
@@ -185,7 +182,6 @@ public Long createJob(String projectUrl,
         Long projectId = projectManager.getProjectId(projectUrl);
         Long runtimeVersionId = runtimeManager.getRuntimeVersionId(runtimeVersionUrl, null);
         Long modelVersionId = swmpManager.getSwmpVersionId(modelVersionUrl, null);
-        Long resourcePoolId = resourcePoolManager.getResourcePoolId(resourcePool);
         JobEntity jobEntity = JobEntity.builder()
                 .ownerId(user.getId())
                 .jobUuid(jobUuid)
@@ -196,7 +192,7 @@ public Long createJob(String projectUrl,
                 .resultOutputPath(storagePathCoordinator.allocateResultMetricsPath(jobUuid))
                 .jobStatus(JobStatus.CREATED)
                 .type(JobType.EVALUATION)
-                .resourcePoolId(resourcePoolId)
+                .resourcePool(resourcePool)
                 .stepSpec(stepSpecOverWrites)
                 .build();
 

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobBoConverter.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobBoConverter.java
@@ -38,16 +38,14 @@
 import ai.starwhale.mlops.domain.swmp.mapper.SwModelPackageMapper;
 import ai.starwhale.mlops.domain.swmp.po.SwModelPackageEntity;
 import ai.starwhale.mlops.domain.system.SystemSettingService;
-import ai.starwhale.mlops.domain.system.mapper.ResourcePoolMapper;
-import ai.starwhale.mlops.domain.system.po.ResourcePoolEntity;
-import ai.starwhale.mlops.domain.system.resourcepool.ResourcePoolConverter;
 import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import ai.starwhale.mlops.domain.task.bo.Task;
 import ai.starwhale.mlops.domain.task.converter.TaskBoConverter;
 import ai.starwhale.mlops.domain.task.mapper.TaskMapper;
 import ai.starwhale.mlops.domain.task.po.TaskEntity;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
@@ -69,10 +67,6 @@ public class JobBoConverter {
 
     final RuntimeVersionMapper runtimeVersionMapper;
 
-    final ResourcePoolMapper resourcePoolMapper;
-
-    final ResourcePoolConverter resourcePoolConverter;
-
     final SwdsBoConverter swdsBoConverter;
 
     final SystemSettingService systemSettingService;
@@ -93,8 +87,6 @@ public JobBoConverter(
             RuntimeMapper runtimeMapper,
             RuntimeVersionMapper runtimeVersionMapper,
             SwdsBoConverter swdsBoConverter,
-            ResourcePoolMapper resourcePoolMapper,
-            ResourcePoolConverter resourcePoolConverter,
             SwmpVersionConvertor swmpVersionConvertor,
             SystemSettingService systemSettingService, StepMapper stepMapper,
             StepConverter stepConverter, TaskMapper taskMapper,
@@ -104,8 +96,6 @@ public JobBoConverter(
         this.runtimeMapper = runtimeMapper;
         this.runtimeVersionMapper = runtimeVersionMapper;
         this.swdsBoConverter = swdsBoConverter;
-        this.resourcePoolMapper = resourcePoolMapper;
-        this.resourcePoolConverter = resourcePoolConverter;
         this.systemSettingService = systemSettingService;
         this.swmpVersionConvertor = swmpVersionConvertor;
         this.stepMapper = stepMapper;
@@ -124,8 +114,6 @@ public Job fromEntity(JobEntity jobEntity) {
                 jobEntity.getRuntimeVersionId());
         RuntimeEntity runtimeEntity = runtimeMapper.findRuntimeById(
                 runtimeVersionEntity.getRuntimeId());
-        ResourcePoolEntity resourcePoolEntity = resourcePoolMapper.findById(jobEntity.getResourcePoolId());
-        ResourcePool resourcePool = resourcePoolConverter.toResourcePool(resourcePoolEntity);
         String image = runtimeVersionEntity.getImage();
         if (null != systemSettingService.getSystemSetting() && null != systemSettingService.getSystemSetting()
                 .getDockerSetting() && null != systemSettingService.getSystemSetting().getDockerSetting()
@@ -160,7 +148,7 @@ public Job fromEntity(JobEntity jobEntity) {
                 .swDataSets(swDataSets)
                 .outputDir(jobEntity.getResultOutputPath())
                 .uuid(jobEntity.getJobUuid())
-                .resourcePool(resourcePool)
+                .resourcePool(systemSettingService.queryResourcePool(jobEntity.getResourcePool()))
                 .build();
         return fillStepsAndTasks(job);
     }

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConvertor.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/converter/JobConvertor.java
@@ -24,9 +24,8 @@
 import ai.starwhale.mlops.domain.job.po.JobEntity;
 import ai.starwhale.mlops.domain.runtime.RuntimeService;
 import ai.starwhale.mlops.domain.swds.po.SwDatasetVersionEntity;
-import ai.starwhale.mlops.domain.system.mapper.ResourcePoolMapper;
-import ai.starwhale.mlops.domain.system.po.ResourcePoolEntity;
-import ai.starwhale.mlops.domain.system.resourcepool.ResourcePoolConverter;
+import ai.starwhale.mlops.domain.system.SystemSettingService;
+import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import ai.starwhale.mlops.domain.user.UserConvertor;
 import ai.starwhale.mlops.exception.ConvertException;
 import ai.starwhale.mlops.exception.SwProcessException;
@@ -41,21 +40,19 @@ public class JobConvertor implements Convertor<JobEntity, JobVo> {
 
     private final IdConvertor idConvertor;
     private final UserConvertor userConvertor;
-    private final ResourcePoolConverter resourcePoolConverter;
     private final RuntimeService runtimeService;
     private final JobSwdsVersionMapper jobSwdsVersionMapper;
-    private final ResourcePoolMapper resourcePoolMapper;
+
+    private final SystemSettingService systemSettingService;
 
     public JobConvertor(IdConvertor idConvertor, UserConvertor userConvertor,
-            ResourcePoolConverter resourcePoolConverter,
             RuntimeService runtimeService, JobSwdsVersionMapper jobSwdsVersionMapper,
-            ResourcePoolMapper resourcePoolMapper) {
+            SystemSettingService systemSettingService) {
         this.idConvertor = idConvertor;
         this.userConvertor = userConvertor;
-        this.resourcePoolConverter = resourcePoolConverter;
         this.runtimeService = runtimeService;
         this.jobSwdsVersionMapper = jobSwdsVersionMapper;
-        this.resourcePoolMapper = resourcePoolMapper;
+        this.systemSettingService = systemSettingService;
     }
 
     @Override
@@ -72,9 +69,6 @@ public JobVo convert(JobEntity jobEntity) throws ConvertException {
                 .map(SwDatasetVersionEntity::getVersionName)
                 .collect(Collectors.toList());
 
-        ResourcePoolEntity resourcePoolEntity = resourcePoolMapper.findById(jobEntity.getResourcePoolId());
-        var resourcePool = resourcePoolConverter.toResourcePool(resourcePoolEntity);
-
         return JobVo.builder()
                 .id(idConvertor.convert(jobEntity.getId()))
                 .uuid(jobEntity.getJobUuid())
@@ -87,7 +81,7 @@ public JobVo convert(JobEntity jobEntity) throws ConvertException {
                 .jobStatus(jobEntity.getJobStatus())
                 .stopTime(jobEntity.getFinishedTime().getTime())
                 .comment(jobEntity.getComment())
-                .resourcePool(resourcePool.getLabel())
+                .resourcePool(systemSettingService.queryResourcePool(jobEntity.getResourcePool()).getName())
                 .build();
     }
 

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/po/JobEntity.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/po/JobEntity.java
@@ -72,7 +72,7 @@ public class JobEntity extends BaseEntity {
 
     private JobType type;
 
-    private Long resourcePoolId;
+    private String resourcePool;
 
     private String stepSpec;
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/bo/Step.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/bo/Step.java
@@ -47,6 +47,4 @@ public class Step extends TimeConcern {
     StepStatus status;
 
     List<Task> tasks;
-
-    List<RuntimeResource> runtimeResources;
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/system/SystemService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/system/SystemService.java
@@ -16,42 +16,29 @@
 
 package ai.starwhale.mlops.domain.system;
 
-import ai.starwhale.mlops.api.protocol.system.ResourcePoolVo;
-import ai.starwhale.mlops.domain.system.mapper.ResourcePoolMapper;
-import ai.starwhale.mlops.domain.system.resourcepool.ResourcePoolConverter;
-import ai.starwhale.mlops.storage.StorageAccessService;
+import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import java.util.List;
-import java.util.stream.Collectors;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 @Service
 public class SystemService {
 
-    private final ResourcePoolMapper resourcePoolMapper;
-
-    private final ResourcePoolConverter resourcePoolConverter;
+    private final SystemSettingService systemSettingService;
 
     private final String controllerVersion;
 
-    private final StorageAccessService storageAccessService;
-
-    public SystemService(ResourcePoolMapper resourcePoolMapper,
-            ResourcePoolConverter resourcePoolConverter, @Value("${sw.version}") String controllerVersion,
-            StorageAccessService storageAccessService) {
-        this.resourcePoolMapper = resourcePoolMapper;
-        this.resourcePoolConverter = resourcePoolConverter;
+    public SystemService(SystemSettingService systemSettingService, @Value("${sw.version}") String controllerVersion) {
+        this.systemSettingService = systemSettingService;
         this.controllerVersion = controllerVersion;
-        this.storageAccessService = storageAccessService;
     }
 
     public String controllerVersion() {
         return controllerVersion;
     }
 
-    public List<ResourcePoolVo> listResourcePools() {
-        var entities = resourcePoolMapper.listResourcePools();
-        return entities.stream().map(resourcePoolConverter::toResourcePoolVo).collect(Collectors.toList());
+    public List<ResourcePool> listResourcePools() {
+        return systemSettingService.getResourcePools();
     }
 
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/system/SystemSetting.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/system/SystemSetting.java
@@ -16,9 +16,12 @@
 
 package ai.starwhale.mlops.domain.system;
 
+import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import java.util.ArrayList;
+import java.util.List;
 import lombok.Data;
 
 @Data
@@ -28,6 +31,8 @@ public class SystemSetting {
 
     DockerSetting dockerSetting;
 
+    List<ResourcePool> resourcePoolSetting = new ArrayList<>();
+
     @Data
     public static class DockerSetting {