Merge branch 'main' into topic/miroslavi/add-security-context-to-data…

…-job-template

projects/control-service/CHANGELOG.md

@@ -11,6 +11,10 @@ MAJOR.MINOR - dd.MM.yyyy
 
 * **Breaking Changes**
 
+1.3 - 18.02.2022
+----
+* **Improvement**
+  * Support rootless data job deployment container images and builder jobs
 
 1.3 - 27.01.2022
 ----

projects/control-service/projects/base/src/main/java/com/vmware/taurus/security/SecurityConfiguration.java

@@ -95,11 +95,11 @@ protected void configure(HttpSecurity http) throws Exception {
     public void configure(WebSecurity web) {
         web.ignoring().antMatchers(
                 "/",
-                "/v2/api-docs",
-                "/swagger-resources/**",
-                "/configuration/**",
-                "/swagger-ui.html",
-                "/webjars/**",
+                "/data-jobs/v2/api-docs",
+                "/data-jobs/swagger-resources/**",
+//                "/data-jobs/configuration/**",
+                "/data-jobs/swagger-ui.html",
+                "/data-jobs/webjars/**",
                 // There should not be sensitive data in prometheus, and it makes
                 // integration with the monitoring system easier if no auth is necessary.
                 "/data-jobs/debug/prometheus",

projects/control-service/projects/helm_charts/pipelines-control-service/templates/NOTES.txt

@@ -15,15 +15,15 @@ To access Data Jobs API from outside your K8s cluster, follow the steps below:
 {{- if contains "NodePort" .Values.service.type }}
 
   export SERVICE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo "Data Jobs API URL: http://$SERVICE_IP:{{ .Values.service.internalPort }}/swagger-ui.html"
+  echo "Data Jobs API URL: http://$SERVICE_IP:{{ .Values.service.internalPort }}/data-jobs/swagger-ui.html"
 
 {{- else if contains "LoadBalancer" .Values.service.type }}
 
   NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ .Release.Name }}-svc'
 
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ .Release.Name }}-svc --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo "Data Jobs API URL: http://$SERVICE_IP:{{ .Values.service.internalPort }}/swagger-ui.html"
+  echo "Data Jobs API URL: http://$SERVICE_IP:{{ .Values.service.internalPort }}/data-jobs/swagger-ui.html"
 
 {{- else }}
 

projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml

@@ -25,7 +25,7 @@ image:
 deploymentBuilderImage:
   registry: registry.hub.docker.com/versatiledatakit
   repository: job-builder
-  tag: "1.2.2"
+  tag: "1.2.3"
 
 
 ## String to partially override pipelines-control-service.fullname template (will maintain the release name)

projects/control-service/projects/job-builder-rootless/Dockerfile.python.vdk

@@ -4,15 +4,21 @@ ARG base_image=python:3.9-slim
 
 FROM $base_image
 
+ARG UID=1000
+ARG GID=1000
+
 # Set the working directory
 WORKDIR /job
 
-# Make sure base image is python based
+# Validate base image is python based
 RUN python -V
+# Create necessary users and set home directory to /job
+RUN groupadd -r -g $GID vdkgroup && useradd -u $UID -g $GID -r vdkuser && chown -R $UID:$GID /job
+ENV HOME=/job
 
 # Copy the actual job that has to be executed
 ARG job_name
-COPY $job_name $job_name/
+COPY --chown=$UID:$GID $job_name $job_name/
 
 # TODO: this would trigger for any change in job even if requirements.txt does not change
 # but there's no COPY_IF_EXISTS command in docker to try copy it.
@@ -22,3 +28,5 @@ RUN if [ -f "$job_name/$requirements_file" ]; then pip3 install --disable-pip-ve
 ARG job_githash
 ENV JOB_NAME $job_name
 ENV VDK_JOB_GITHASH $job_githash
+
+USER $UID

projects/control-service/projects/job-builder-rootless/version.txt

@@ -1 +1 @@
-1.3.0dev2
+1.3.1dev2

projects/control-service/projects/job-builder/Dockerfile.python.vdk

@@ -4,15 +4,21 @@ ARG base_image=python:3.9-slim
 
 FROM $base_image
 
+ARG UID=1000
+ARG GID=1000
+
 # Set the working directory
 WORKDIR /job
 
-# Make sure base image is python based
+# Validate base image is python based
 RUN python -V
+# Create necessary users and set home directory to /job
+RUN groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job
+ENV HOME=/job
 
 # Copy the actual job that has to be executed
 ARG job_name
-COPY $job_name $job_name/
+COPY --chown=$UID:$GID $job_name $job_name/
 
 # TODO: this would trigger for any change in job even if requirements.txt does not change
 # but there's no COPY_IF_EXISTS command in docker to try copy it.
@@ -22,3 +28,5 @@ RUN if [ -f "$job_name/$requirements_file" ]; then pip3 install --disable-pip-ve
 ARG job_githash
 ENV JOB_NAME $job_name
 ENV VDK_JOB_GITHASH $job_githash
+
+USER $UID

projects/control-service/projects/job-builder/version.txt

@@ -1 +1 @@
-1.2.2
+1.2.3

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/DataJobCrudIT.java

@@ -68,7 +68,31 @@ public void testDataJobCrud() throws Exception {
       testDataJobPostCreateWebHooks();
 
       // Execute get swagger with no user
-      mockMvc.perform(get("/swagger-ui.html")
+      mockMvc.perform(get("/data-jobs/swagger-ui.html")
+              .content(dataJobRequestBody)
+              .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui.css.map")
+                      .content(dataJobRequestBody)
+                      .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/webjars/springfox-swagger-ui/swagger-ui-bundle.js.map")
+                      .content(dataJobRequestBody)
+                      .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/swagger-resources/configuration/ui")
+                      .content(dataJobRequestBody)
+                      .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/swagger-resources/configuration/security")
+                      .content(dataJobRequestBody)
+                      .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/swagger-resources")
+                      .content(dataJobRequestBody)
+                      .contentType(MediaType.APPLICATION_JSON))
+              .andExpect(status().isOk());
+      mockMvc.perform(get("/data-jobs/v2/api-docs")
                       .content(dataJobRequestBody)
                       .contentType(MediaType.APPLICATION_JSON))
               .andExpect(status().isOk());

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLExecutionsIT.java

@@ -54,9 +54,18 @@ public class GraphQLExecutionsIT extends BaseIT {
    public void setup() {
       cleanup();
 
-      this.dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, new JobConfig()));
-      this.dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, new JobConfig()));
-      this.dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, new JobConfig()));
+      var config1 = new JobConfig();
+      config1.setTeam("test-team1");
+
+      var config2 = new JobConfig();
+      config2.setTeam("test-team2");
+
+      var config3 = new JobConfig();
+      config3.setTeam("test-team3");
+
+      this.dataJob1 = jobsRepository.save(new DataJob(TEST_JOB_NAME_1, config1));
+      this.dataJob2 = jobsRepository.save(new DataJob(TEST_JOB_NAME_2, config2));
+      this.dataJob3 = jobsRepository.save(new DataJob(TEST_JOB_NAME_3, config3));
 
       OffsetDateTime now = OffsetDateTime.now();
       this.dataJobExecution1 = JobExecutionUtil.createDataJobExecution(
@@ -297,4 +306,31 @@ public void testExecutions_filterByJobNameIn() throws Exception {
             .andExpect(jsonPath("$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution2.getId()))));
    }
 
+   @Test
+   public void testExecutions_filterByTeamNameIn() throws Exception {
+      mockMvc.perform(MockMvcRequestBuilders.get(JOBS_URI)
+                  .queryParam("query", getQuery())
+                  .param("variables", "{" +
+                        "\"filter\": {" +
+                        "      \"teamNameIn\": [\"" + dataJobExecution1.getDataJob().getJobConfig().getTeam() + "\"]" +
+                        "    }," +
+                        "\"pageNumber\": 1," +
+                        "\"pageSize\": 10" +
+                        "}")
+                  .with(user("user")))
+            .andExpect(status().is(200))
+            .andExpect(content().contentType("application/json"))
+            .andExpect(jsonPath(
+                  "$.data.content[*].id",
+                  Matchers.contains(dataJobExecution1.getId())))
+            .andExpect(jsonPath(
+                  "$.data.content[*].jobName",
+                  Matchers.contains(dataJob1.getName())))
+            .andExpect(jsonPath(
+                  "$.data.content[*].status",
+                  Matchers.contains(dataJobExecution1.getStatus().toString())))
+            .andExpect(jsonPath("$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution3.getId()))))
+            .andExpect(jsonPath("$.data.content[*].id", Matchers.not(Matchers.contains(dataJobExecution2.getId()))));
+   }
+
 }

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/graphql/it/GraphQLJobExecutionsStatusCountIT.java

@@ -5,7 +5,6 @@
 
 package com.vmware.taurus.graphql.it;
 
-import com.vmware.taurus.ServiceApp;
 import com.vmware.taurus.datajobs.it.common.BaseDataJobDeploymentIT;
 import com.vmware.taurus.service.JobExecutionRepository;
 import com.vmware.taurus.service.JobsRepository;
@@ -14,12 +13,8 @@
 import com.vmware.taurus.service.model.ExecutionStatus;
 import com.vmware.taurus.service.model.ExecutionType;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
-import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
 
 import java.time.OffsetDateTime;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/JobExecutionFilterSpec.java

@@ -9,6 +9,7 @@
 import com.vmware.taurus.service.model.DataJobExecution;
 import com.vmware.taurus.service.model.DataJobExecution_;
 import com.vmware.taurus.service.model.DataJob_;
+import com.vmware.taurus.service.model.JobConfig_;
 import lombok.AllArgsConstructor;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.data.jpa.domain.Specification;
@@ -53,6 +54,11 @@ public Predicate toPredicate(Root<DataJobExecution> root, CriteriaQuery<?> query
          if (CollectionUtils.isNotEmpty(filter.getJobNameIn())) {
             predicates.add(root.get(DataJobExecution_.DATA_JOB).get(DataJob_.NAME).in(filter.getJobNameIn()));
          }
+
+         if (CollectionUtils.isNotEmpty(filter.getTeamNameIn())) {
+            predicates.add(root.get(DataJobExecution_.DATA_JOB).get(DataJob_.JOB_CONFIG).get(JobConfig_.TEAM).in(filter.getTeamNameIn()));
+         }
+
       }
 
       return builder.and(predicates.toArray(new Predicate[0]));

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/doc/SwaggerConfig.java

@@ -11,6 +11,8 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Controller;
+import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
+import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 import springfox.documentation.builders.PathSelectors;
 import springfox.documentation.builders.RequestHandlerSelectors;
@@ -38,6 +40,7 @@
 @EnableSwagger2
 @Controller
 public class SwaggerConfig implements WebMvcConfigurer {
+    private static final String PATH = "/data-jobs";
     private static final String AUTHORIZE_KEY_NAME = "Authorization Header (put 'Bearer access_token')";
 
     @Bean
@@ -56,6 +59,25 @@ public Docket swaggerSpringMvcPlugin(ServletContext context) {
                         new Tag("Data Jobs Sources", "(Stable)"));
     }
 
+
+    @Override
+    public void addViewControllers(ViewControllerRegistry registry) {
+        final var apiDocs = "/v2/api-docs";
+        final var configUi = "/swagger-resources/configuration/ui";
+        final var configSecurity = "/swagger-resources/configuration/security";
+        final var resources = "/swagger-resources";
+
+        registry.addViewController(PATH + apiDocs).setViewName("forward:" + apiDocs);
+        registry.addViewController(PATH + configUi).setViewName("forward:" + configUi);
+        registry.addViewController(PATH + configSecurity).setViewName("forward:" + configSecurity);
+        registry.addViewController(PATH + resources).setViewName("forward:" + resources);
+    }
+
+    @Override
+    public void addResourceHandlers(ResourceHandlerRegistry registry) {
+        registry.addResourceHandler(PATH + "/**").addResourceLocations("classpath:/META-INF/resources/");
+    }
+
     // springfox does not support Bearer Token Auth from api.yaml so we hack it manually
     // TODO: there must be cleaner way. We may consider switch to other library also.
     private ApiKey apiKey() {

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/graphql/ExecutionDataFetcher.java

@@ -215,6 +215,7 @@ private static Optional<DataJobExecutionFilter> extractDataJobExecutionFilter(Ma
                      .startTimeLte((OffsetDateTime) filterRaw.get(DataJobExecutionFilter.START_TIME_LTE_FIELD))
                      .endTimeLte((OffsetDateTime) filterRaw.get(DataJobExecutionFilter.END_TIME_LTE_FIELD))
                      .jobNameIn((List<String>) filterRaw.get(DataJobExecutionFilter.JOB_NAME_IN_FIELD))
+                     .teamNameIn((List<String>) filterRaw.get(DataJobExecutionFilter.TEAM_NAME_IN_FIELD))
                      .build()
          );
       }

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/graphql/model/DataJobExecutionFilter.java

@@ -24,11 +24,13 @@ public class DataJobExecutionFilter {
    public static final String JOB_NAME_IN_FIELD = "jobNameIn";
    public static final String START_TIME_LTE_FIELD = "startTimeLte";
    public static final String END_TIME_LTE_FIELD = "endTimeLte";
+   public static final String TEAM_NAME_IN_FIELD = "teamNameIn";
 
    private OffsetDateTime startTimeGte;
    private OffsetDateTime endTimeGte;
    private List<ExecutionStatus> statusIn;
    private List<String> jobNameIn;
+   private List<String> teamNameIn;
    private OffsetDateTime startTimeLte;
    private OffsetDateTime endTimeLte;
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/graphql/model/DataJobExecutionOrder.java

@@ -7,6 +7,7 @@
 
 import com.vmware.taurus.service.model.DataJobExecution_;
 import com.vmware.taurus.service.model.DataJob_;
+import com.vmware.taurus.service.model.JobConfig_;
 import lombok.Builder;
 import lombok.Data;
 import org.springframework.data.domain.Sort;
@@ -19,6 +20,7 @@
 public class DataJobExecutionOrder {
 
    public static final String DATA_JOB_NAME = "jobName";
+   public static final String DATA_JOB_TEAM = "jobTeam";
 
    public static final Set<String> AVAILABLE_PROPERTIES = Set.of(
          DataJobExecution_.MESSAGE,
@@ -34,10 +36,12 @@ public class DataJobExecutionOrder {
          DataJobExecution_.STARTED_BY,
          DataJobExecution_.STATUS,
          DataJobExecution_.VDK_VERSION,
-         DATA_JOB_NAME);
+         DATA_JOB_NAME,
+         DATA_JOB_TEAM);
 
    public static final Map<String, String> PUBLIC_NAME_TO_DB_ENTITY_MAP = Map.of(
-         DATA_JOB_NAME, DataJobExecution_.DATA_JOB + "." + DataJob_.NAME
+         DATA_JOB_NAME, DataJobExecution_.DATA_JOB + "." + DataJob_.NAME,
+         DATA_JOB_TEAM, DataJobExecution_.DATA_JOB + "." + DataJob_.JOB_CONFIG + "." + JobConfig_.TEAM
    );
 
    public static final String PROPERTY_FIELD = "property";

projects/control-service/projects/pipelines_control_service/src/main/resources/schema.graphqls

@@ -25,6 +25,7 @@ input DataJobExecutionFilter {
     endTimeGte: DateTime,
     statusIn: [DataJobExecutionStatus],
     jobNameIn: [String],
+    teamNameIn: [String],
     startTimeLte: DateTime,
     endTimeLte: DateTime
 }

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/RepositoryUtil.java

@@ -23,17 +23,22 @@ public static DataJob createDataJob(JobsRepository jobsRepository) {
       return createDataJob(jobsRepository, "test-job");
    }
 
-   public static DataJob createDataJob(JobsRepository jobsRepository, String jobName) {
+   public static DataJob createDataJob(JobsRepository jobsRepository, String jobName){
+      return createDataJob(jobsRepository, jobName, "test-team");
+   }
+
+   public static DataJob createDataJob(JobsRepository jobsRepository, String jobName, String jobTeam) {
       JobConfig config = new JobConfig();
       config.setSchedule("schedule");
-      config.setTeam("test-team");
+      config.setTeam(jobTeam);
       var expectedJob = new DataJob(jobName, config, DeploymentStatus.NONE);
       var actualJob = jobsRepository.save(expectedJob);
       Assertions.assertEquals(expectedJob, actualJob);
 
       return actualJob;
    }
 
+
    public static DataJobExecution createDataJobExecution(
          JobExecutionRepository jobExecutionRepository,
          String executionId,