perf(titus): Support for loading minimal clusters

This allows more optimal server group from cluster lookups where filtering is applied to `cluster.serverGroups`. Only a matched server group will be expanded. This approach offers significant improvements when dealing with clusters containing > 1 server group. xxx
spinnaker · Mar 26, 2018 · 134300a · 134300a
1 parent cb3b2bf
commit 134300a
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 21 deletions.
diff --git a/...vy/com/netflix/spinnaker/clouddriver/titus/caching/agents/TitusClusterCachingAgent.groovy b/...vy/com/netflix/spinnaker/clouddriver/titus/caching/agents/TitusClusterCachingAgent.groovy
@@ -209,15 +209,19 @@ class TitusClusterCachingAgent implements CachingAgent, CustomScheduledAgent {
         ]
       } : []
 
+      // tasks are cached independently as instances so avoid the overhead of also storing on the serialized job
+      def jobTasks = job.tasks
+      job.tasks = []
+
       attributes.job = job
       attributes.scalingPolicies = policies
-      attributes.tasks = data.job.tasks
+      attributes.tasks = jobTasks.collect { [ id: it.id, instanceId: it.instanceId ] }
       attributes.region = region
       attributes.account = account.name
       relationships[APPLICATIONS.ns].add(data.appName)
       relationships[CLUSTERS.ns].add(data.cluster)
       relationships[INSTANCES.ns].addAll(data.instanceIds)
-      for (Job.TaskSummary task : job.tasks) {
+      for (Job.TaskSummary task : jobTasks) {
         def instanceData = new InstanceData(job, task, account.name, region, account.stack)
         cacheInstance(instanceData, instances)
       }

diff --git a/...ovy/com/netflix/spinnaker/clouddriver/titus/caching/providers/TitusClusterProvider.groovy b/...ovy/com/netflix/spinnaker/clouddriver/titus/caching/providers/TitusClusterProvider.groovy
@@ -168,7 +168,7 @@ class TitusClusterProvider implements ClusterProvider<TitusCluster> {
 
   @Override
   boolean supportsMinimalClusters() {
-    return false
+    return true
   }
 
   // Private methods
@@ -183,29 +183,16 @@ class TitusClusterProvider implements ClusterProvider<TitusCluster> {
    * Translate clusters
    */
   private Collection<TitusCluster> translateClusters(Collection<CacheData> clusterData, boolean includeDetails) {
-    Map<String, TitusServerGroup> serverGroups
-    if (includeDetails) {
-      Collection<CacheData> allServerGroups = resolveRelationshipDataForCollection(clusterData, SERVER_GROUPS.ns, RelationshipCacheFilter.include(INSTANCES.ns))
-      serverGroups = translateServerGroups(allServerGroups)
-    }
+    def relationshipFilter = includeDetails ? RelationshipCacheFilter.include(INSTANCES.ns) : RelationshipCacheFilter.none()
+    Collection<CacheData> allServerGroups = resolveRelationshipDataForCollection(clusterData, SERVER_GROUPS.ns, relationshipFilter)
+    Map<String, TitusServerGroup> serverGroups = translateServerGroups(allServerGroups)
+
     Collection<TitusCluster> clusters = clusterData.collect { CacheData clusterDataEntry ->
       Map<String, String> clusterKey = Keys.parse(clusterDataEntry.id)
       TitusCluster cluster = new TitusCluster()
       cluster.accountName = clusterKey.account
       cluster.name = clusterKey.cluster
-      if (includeDetails) {
-        cluster.serverGroups = clusterDataEntry.relationships[SERVER_GROUPS.ns]?.findResults { serverGroups.get(it) }
-      } else {
-        cluster.serverGroups = clusterDataEntry.relationships[SERVER_GROUPS.ns]?.collect { serverGroupKey ->
-          Map<String, String> keyParts = Keys.parse(serverGroupKey)
-          TitusServerGroup titusServerGroup = new TitusServerGroup()
-          titusServerGroup.placement.account = keyParts.account
-          titusServerGroup.placement.region = keyParts.region
-          titusServerGroup.application = keyParts.application
-          titusServerGroup.name = keyParts.serverGroup
-          titusServerGroup
-        }
-      }
+      cluster.serverGroups = clusterDataEntry.relationships[SERVER_GROUPS.ns]?.findResults { serverGroups.get(it) }
       cluster
     }
     clusters
@@ -223,6 +210,14 @@ class TitusClusterProvider implements ClusterProvider<TitusCluster> {
       Job job = objectMapper.readValue(json, Job)
       TitusServerGroup serverGroup = new TitusServerGroup(job, serverGroupEntry.attributes.account, serverGroupEntry.attributes.region)
       serverGroup.instances = serverGroupEntry.relationships[INSTANCES.ns]?.findResults { instances.get(it) } as Set
+
+      if (!serverGroup.instances && serverGroupEntry.attributes.tasks) {
+        // has no direct instance relationships but we can partially populate instances based on attributes.tasks
+        serverGroup.instances = serverGroupEntry.attributes.tasks.collect {
+          new TitusInstance(it as Map)
+        }
+      }
+
       [(serverGroupEntry.id): serverGroup]
     }
     serverGroups

diff --git a/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/model/TitusInstance.groovy b/src/main/groovy/com/netflix/spinnaker/clouddriver/titus/model/TitusInstance.groovy
@@ -44,6 +44,8 @@ class TitusInstance implements Instance {
   final String providerType = TitusCloudProvider.ID
   final String cloudProvider = TitusCloudProvider.ID
 
+  TitusInstance() {}
+
   TitusInstance(Job job, Job.TaskSummary task) {
     id = task.id
     instanceId = task.instanceId