fix(controller): fix integeration test bugs (#1025)

fix token bug;fix dataset uri env bug; fix ; fix key prefix bug
star-whale · Aug 28, 2022 · c9cbd98 · c9cbd98
1 parent e815a69
commit c9cbd98
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 21 deletions.
diff --git a/...controller-sdk/src/main/java/ai/starwhale/mlops/api/protocol/report/resp/TaskRequest.java b/...controller-sdk/src/main/java/ai/starwhale/mlops/api/protocol/report/resp/TaskRequest.java
@@ -28,10 +28,6 @@
 @AllArgsConstructor
 @NoArgsConstructor
 public class TaskRequest {
-    String jobId;
-    String project;
     Integer index;
     Integer total;
-    String stepName;
-    List<String> datasetUris;
 }
diff --git a/...er/controller/src/main/java/ai/starwhale/mlops/configuration/security/JobTokenConfig.java b/...er/controller/src/main/java/ai/starwhale/mlops/configuration/security/JobTokenConfig.java
@@ -47,6 +47,6 @@ public JobTokenConfig(JwtTokenUtil jwtTokenUtil,
     @Override
     public void run(String... args) throws Exception {
         User user = userService.loadUserById(jobUserId);
-        this.token = jwtTokenUtil.generateAccessToken(user,null);
+        this.token = String.format("Bearer %s",jwtTokenUtil.generateAccessToken(user,null));
     }
 }
diff --git a/...ontroller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java b/...ontroller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java
@@ -153,13 +153,7 @@ public List<StepEntity> split(Job job) {
                     .stepId(stepEntity.getId())
                     .taskRequest(JSONUtil.toJsonStr(
                             TaskRequest.builder()
-                                .project(job.getProject().getName())
-                                .jobId(job.getUuid())
-                                .stepName(stepEntity.getName())
                                 .total(stepEntity.getTaskNum())
-                                .datasetUris(job.getSwDataSets().stream()
-                                    .map(ds -> String.format("%s/version/%s", ds.getName(), ds.getVersion()))
-                                    .collect(Collectors.toList()))
                                 .index(i)
                                 .build()
                         )
@@ -211,8 +205,4 @@ private String storagePath(String jobId, String taskId) {
         return storagePathCoordinator.generateTaskResultPath(jobId, taskId);
     }
 
-    public static void main(String[] args) {
-        String s = JSONUtil.toJsonStr(TaskRequest.builder().project("p").datasetUris(List.of("rwe")).jobId("1l").build());
-        System.out.println(s);
-    }
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/schedule/k8s/K8sTaskScheduler.java b/server/controller/src/main/java/ai/starwhale/mlops/schedule/k8s/K8sTaskScheduler.java
@@ -111,6 +111,10 @@ public void remove(Collection<Long> taskIds) {
         });
     }
 
+    /**
+     * {instance}/project/{projectName}/dataset/{datasetName}/version/{version}
+     */
+    static final String FORMATTER_URI_DATASET="%s/project/%s/dataset/%s/version/%s";
     /**
      * todo hard code in this piece of code will be refactored after other core concepts being refactored
      *
@@ -140,24 +144,26 @@ private void deployTaskToK8s(K8sClient client, String image, Task task, Map<Stri
         initContainerEnvs.put("SW_PYPI_TRUSTED_HOST",runTimeProperties.getPypi().getTrustedHost());
         // task container envs
         Map<String, String> coreContainerEnvs = new HashMap<>();
-        coreContainerEnvs.put("SW_TASK_STEP", task.getTaskRequest().getStepName());
+        coreContainerEnvs.put("SW_TASK_STEP", task.getStep().getName());
         // TODO: support multi dataset uris
-        coreContainerEnvs.put("SW_DATASET_URI", task.getTaskRequest().getDatasetUris().get(0));
-        coreContainerEnvs.put("SW_TASK_INDEX", String.valueOf(task.getTaskRequest().getIndex()));
-        coreContainerEnvs.put("SW_EVALUATION_VERSION", task.getTaskRequest().getJobId());
         // oss env
         List<SWDataSet> swDataSets = swJob.getSwDataSets();
+        SWDataSet swDataSet = swDataSets.get(0);
+        coreContainerEnvs.put("SW_DATASET_URI", String.format(FORMATTER_URI_DATASET,instanceUri,swJob.getProject().getName(),swDataSet.getName(),swDataSet.getVersion()));
+        coreContainerEnvs.put("SW_TASK_INDEX", String.valueOf(task.getTaskRequest().getIndex()));
+        coreContainerEnvs.put("SW_EVALUATION_VERSION", swJob.getId().toString());
+
         swDataSets.forEach(ds -> ds.getFileStorageEnvs().values()
             .forEach(fileStorageEnv -> coreContainerEnvs.putAll(fileStorageEnv.getEnvs())));
 
-        coreContainerEnvs.put(FileStorageEnv.ENV_KEY_PREFIX, storagePathCoordinator.getSwdsPathNamedFormatter());
+        coreContainerEnvs.put(FileStorageEnv.ENV_KEY_PREFIX, swDataSet.getPath());
 //        coreContainerEnvs.put("SW_S3_READ_TIMEOUT", );
 //        coreContainerEnvs.put("SW_S3_TOTAL_MAX_ATTEMPTS", );
 
         // datastore env
         coreContainerEnvs.put("SW_TOKEN", jobTokenConfig.getToken());
         coreContainerEnvs.put("SW_INSTANCE_URI", instanceUri);
-        coreContainerEnvs.put("SW_PROJECT", task.getTaskRequest().getProject());
+        coreContainerEnvs.put("SW_PROJECT", swJob.getProject().getName());
         try {
             // cmd（all、single[step、taskIndex]）
             String cmd = "run";