Titus v3 api integration

clouddriver-titus.gradle

@@ -1,7 +1,27 @@
+apply plugin: 'com.google.protobuf'
+
+buildscript {
+  dependencies {
+    classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.0'
+  }
+}
+
+ext {
+  protobufVersion = '3.2.0'
+  grpcVersion = '1.2.+'
+  netflixGrpcVersion = '1.0.+'
+}
+
 dependencies {
-  compile("com.netflix.spinnaker.clouddriver:clouddriver-core:${clouddriverVersion}")
-  compile("com.netflix.spinnaker.clouddriver:clouddriver-eureka:${clouddriverVersion}")
-  compile("com.netflix.spinnaker.clouddriver:clouddriver-aws:${clouddriverVersion}")
+  protobuf group: 'netflix', name: 'titus-api-definitions', version: "1.14"
+
+  compile "com.google.protobuf:protobuf-java:$protobufVersion"
+  compile "io.grpc:grpc-protobuf:$grpcVersion"
+  compile "io.grpc:grpc-stub:$grpcVersion"
+  compile "io.grpc:grpc-netty:$grpcVersion"
+  compile "com.netflix.spinnaker.clouddriver:clouddriver-core:${clouddriverVersion}"
+  compile "com.netflix.spinnaker.clouddriver:clouddriver-eureka:${clouddriverVersion}"
+  compile "com.netflix.spinnaker.clouddriver:clouddriver-aws:${clouddriverVersion}"
   compile spinnaker.dependency('bootActuator')
   compile spinnaker.dependency('bootWeb')
   compile 'com.squareup.retrofit2:retrofit:2.0.0-beta4'
@@ -11,3 +31,40 @@ dependencies {
   testCompile "org.slf4j:slf4j-simple:${spinnaker.version('slf4j')}"
   spinnaker.group('test')
 }
+
+sourceSets {
+  main {
+    java {
+      srcDir "${protobuf.generatedFilesBaseDir}/main/grpc"
+      srcDir "${protobuf.generatedFilesBaseDir}/main/java"
+    }
+  }
+}
+
+task cleanGenerated(type: Delete) {
+  delete "$projectDir/src/generated"
+}
+
+clean.dependsOn cleanGenerated
+
+protobuf {
+  protoc {
+    artifact = "com.google.protobuf:protoc:$protobufVersion"
+  }
+  plugins {
+    grpc {
+      artifact = "io.grpc:protoc-gen-grpc-java:$grpcVersion"
+    }
+    netflix_protoc_wrapper_type {
+      artifact = "netflix.grpc:netflix-protoc-wrapper-type:1.0.0"
+    }
+  }
+  generateProtoTasks {
+    all()*.plugins {
+      grpc {}
+      netflix_protoc_wrapper_type {
+        outputSubDir = "java"
+      }
+    }
+  }
+}

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/TitusClientProvider.groovy

@@ -22,6 +22,7 @@ import com.netflix.spinnaker.clouddriver.titus.client.TitusJobCustomizer
 import com.netflix.spinnaker.clouddriver.titus.client.TitusRegion
 import com.netflix.spinnaker.clouddriver.titus.credentials.NetflixTitusCredentials
 import com.netflix.spinnaker.clouddriver.titus.client.TitusClient
+import com.netflix.spinnaker.clouddriver.titus.v3client.RegionScopedV3TitusClient
 import groovy.transform.Immutable
 
 import java.util.concurrent.ConcurrentHashMap
@@ -40,7 +41,7 @@ class TitusClientProvider {
   TitusClient getTitusClient(NetflixTitusCredentials account, String region) {
     final TitusRegion titusRegion = Objects.requireNonNull(account.regions.find { it.name == region }, "region")
     final TitusClientKey key = new TitusClientKey(Objects.requireNonNull(account.name), titusRegion)
-    return titusClients.computeIfAbsent(key, { k -> new RegionScopedTitusClient(k.region, registry, titusJobCustomizers) })
+    return titusClients.computeIfAbsent(key, { k -> account.version == 'v3' ? new RegionScopedV3TitusClient(k.region, registry, titusJobCustomizers) : new RegionScopedTitusClient(k.region, registry, titusJobCustomizers) })
   }
 
   @Immutable(knownImmutableClasses = [TitusRegion])

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/RegionScopedTitusClient.java

@@ -210,11 +210,6 @@ public String submitJob(SubmitJobRequest submitJobRequest) {
         return jobUri.substring(jobUri.lastIndexOf("/") + 1);
     }
 
-    @Override
-    public void updateJob(String jobId, Map<String, Object> jobAttributes) {
-        execute("updateJob", titusRestAdapter.updateJob(jobId, jobAttributes));
-    }
-
     @Override
     public Task getTask(String taskId) {
         return execute("getTask", titusRestAdapter.getTask(taskId));
@@ -273,14 +268,6 @@ public List<Job> getAllJobs() {
         return getAllJobsStream().collect(Collectors.toList());
     }
 
-    @Override
-    public List<Job.TaskSummary> getAllTasks() {
-
-        return getAllJobsStream()
-                .flatMap(j -> j.getTasks().stream())
-                .collect(Collectors.toList());
-    }
-
     private Stream<Job> getAllJobsStream() {
         return execute("getJobsByType", titusRestAdapter.getJobsByType("service")).stream();
     }

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/TitusClient.java

@@ -49,12 +49,6 @@ public interface TitusClient {
      */
     public String submitJob(SubmitJobRequest submitJobRequest);
 
-    /**
-     *
-     * @param jobAttributes
-     */
-    public void updateJob(String jobId, Map<String, Object> jobAttributes);
-
     /**
      *
      * @param taskId
@@ -103,9 +97,4 @@ public interface TitusClient {
      */
     public List<Job> getAllJobs();
 
-    /**
-     * @return
-     */
-    public List<Job.TaskSummary> getAllTasks();
-
 }

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/Job.java

@@ -201,6 +201,54 @@ public void setSnapshots(String snapshots) {
 
     public Job() {}
 
+    public Job(com.netflix.titus.grpc.protogen.Job grpcJob){
+      id = grpcJob.getId();
+
+      if(grpcJob.getJobDescriptor().getBatch() != null){
+        type = "batch";
+      } else if(grpcJob.getJobDescriptor().getService() != null){
+        type = "service";
+      }
+      /*
+      private String name;
+      private String type;
+      private List<String> tags;
+      private String applicationName;
+      private String appName;
+      private String user;
+      private String version;
+      private String entryPoint;
+      private String iamProfile;
+      private String capacityGroup;
+      private Boolean inService;
+      private int instances;
+      private int instancesMin;
+      private int instancesMax;
+      private int instancesDesired;
+      private int cpu;
+      private int memory;
+      private int disk;
+      private int gpu;
+      private int networkMbps;
+      private int[] ports;
+      private Map<String, String> environment;
+      private int retries;
+      private int runtimeLimitSecs;
+      private boolean allocateIpAddress;
+      private Date submittedAt;
+      private List<TaskSummary> tasks;
+      private Map<String, String> labels;
+      private List<String> securityGroups;
+      private Set<TitusSecurityGroup> securityGroupDetails;
+      private String jobGroupStack;
+      private String jobGroupDetail;
+      private String jobGroupSequence;
+      private List<String> hardConstraints;
+      private List<String> softConstraints;
+      private Efs efs;
+      */
+    }
+
     public String getId() {
         return id;
     }

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/JobDescription.java

@@ -16,7 +16,9 @@
 
 package com.netflix.spinnaker.clouddriver.titus.client.model;
 
-import java.util.ArrayList;
+import com.netflix.titus.grpc.protogen.*;
+import com.sun.net.httpserver.Authenticator;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -330,4 +332,116 @@ public void setLabels(Map<String, String> labels) {
 
     public void setSecurityGroups(List<String> securityGroups) { this.securityGroups = securityGroups; }
 
+    public JobDescriptor getGrpcJobDescriptor() {
+
+      // trying to keep the same order as in the proto definition https://stash.corp.netflix.com/projects/TN/repos/titus-api-definitions/browse/src/main/proto/netflix/titus/titus_job_api.proto
+
+      JobDescriptor.Builder jobDescriptorBuilder = JobDescriptor.newBuilder();
+
+      jobDescriptorBuilder.setOwner(Owner.newBuilder().setTeamEmail(user));
+      jobDescriptorBuilder.setAppName(appName);
+
+      JobGroupInfo.Builder jobGroupInfoBuilder = JobGroupInfo.newBuilder();
+      if (jobGroupStack != null) {
+        jobGroupInfoBuilder.setStack(jobGroupStack);
+      }
+      if (jobGroupDetail != null) {
+        jobGroupInfoBuilder.setDetail(jobGroupDetail);
+      }
+      jobGroupInfoBuilder.setSequence(jobGroupSequence);
+
+      jobDescriptorBuilder.setJobGroupInfo(jobGroupInfoBuilder);
+
+      if(!labels.isEmpty()) {
+        jobDescriptorBuilder.putAllLabels(labels);
+      }
+
+      Container.Builder containerBuilder = Container.newBuilder();
+
+      containerBuilder.setName("main");
+
+      Map<String, String> resources = new HashMap<String, String>();
+
+      if (cpu != 0) {
+        resources.put("cpu", String.valueOf(cpu));
+      }
+
+      if (gpu!=0){
+        resources.put("gpu", String.valueOf(gpu));
+      }
+
+      if(networkMbps!=0){
+        resources.put("network", String.valueOf(networkMbps) + "Mbps");
+      }
+
+      if(memory!=0){
+        resources.put("memory", String.valueOf(memory) + "MB");
+      }
+
+      if(disk!=0){
+        resources.put("disk", String.valueOf(disk) + "MB");
+      }
+
+      if(!resources.isEmpty()){
+        containerBuilder.putAllResources(resources);
+      }
+
+      if(!securityGroups.isEmpty()){
+        containerBuilder.putSecurityProfile("securityGroups", String.join(",", securityGroups));
+      }
+
+      if(iamProfile != null) {
+        containerBuilder.putSecurityProfile("iamProfile", iamProfile);
+      }
+
+      containerBuilder.setImage(Image.newBuilder().setName(applicationName + ":" + version));
+
+      if(entryPoint != null){
+        containerBuilder.addEntryPoint(entryPoint);
+      }
+
+      if(!env.isEmpty()){
+        containerBuilder.putAllEnv(env);
+      }
+
+      if(!softConstraints.isEmpty()){
+        containerBuilder.setSoftConstraints(constraintTransformer(softConstraints));
+      }
+
+      if(!hardConstraints.isEmpty()){
+        containerBuilder.setSoftConstraints(constraintTransformer(hardConstraints));
+      }
+
+      jobDescriptorBuilder.addContainers(containerBuilder);
+
+      TaskInstances.Builder taskInstances = TaskInstances.newBuilder();
+      taskInstances.setMin(instancesMin).setMax(instancesMax).setDesired(instancesDesired);
+
+      if(type == "service"){
+        jobDescriptorBuilder.setService(ServiceJobSpec.newBuilder().setContainerId("main").setEnabled(inService).setInstances(taskInstances));
+      }
+
+      if(type == "batch"){
+        BatchJobSpec.Builder batchJobSpec = BatchJobSpec.newBuilder();
+        batchJobSpec.setContainerId("main").setInstances(taskInstances);
+        if(runtimeLimitSecs != 0){
+          batchJobSpec.setRuntimeLimitSecs(runtimeLimitSecs);
+        }
+        if(retries!=0){
+          batchJobSpec.setRetryPolicy(RetryPolicy.newBuilder().setImmediate(RetryPolicy.Immediate.newBuilder().setRetries(retries)));
+        }
+        jobDescriptorBuilder.setBatch(batchJobSpec);
+      }
+
+      return jobDescriptorBuilder.build();
+    }
+
+    private Constraints.Builder constraintTransformer(List<String> constraints){
+      Constraints.Builder constraintsBuilder = Constraints.newBuilder();
+      constraints.forEach( constraint ->
+        { constraintsBuilder.putConstraints(constraint, "true"); }
+      );
+      return constraintsBuilder;
+    }
+
 }

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/client/model/Task.java

@@ -45,6 +45,10 @@ public class Task {
     private String logs;
     private String snapshots;
 
+    public Task(com.netflix.titus.grpc.protogen.Task grpcTask) {
+      this.id = grpcTask.getId();
+    }
+
     public String getId() {
         return id;
     }

src/main/groovy/com/netflix/spinnaker/clouddriver/titus/credentials/NetflixTitusCredentials.groovy

@@ -33,6 +33,7 @@ class NetflixTitusCredentials implements AccountCredentials<TitusCredentials> {
   final String discovery
   final String awsAccount
   final String awsVpc
+  final String version
   final boolean discoveryEnabled
   final String stack
 
@@ -48,7 +49,8 @@ class NetflixTitusCredentials implements AccountCredentials<TitusCredentials> {
                           String awsVpc,
                           boolean discoveryEnabled,
                           String discovery,
-                          String stack
+                          String stack,
+                          String version
   ) {
     this.name = name
     this.environment = environment
@@ -61,6 +63,7 @@ class NetflixTitusCredentials implements AccountCredentials<TitusCredentials> {
     this.discoveryEnabled = discoveryEnabled
     this.discovery = discovery
     this.stack = stack
+    this.version = version
   }
 
   @Override
@@ -97,6 +100,10 @@ class NetflixTitusCredentials implements AccountCredentials<TitusCredentials> {
     return discoveryEnabled
   }
 
+  String getVersion(){
+    return version
+  }
+
   String getStack() {
     return stack
   }