perf(kubernetes): Fix all agents handling on-demand requests (#4110)

* test(kubernetes): Add tests to KubernetesCoreCachingAgent

The next commit will fix a bug in the Kubernetes caching agents;
there are currently no tests validating these agents. Add some tests
on the KubernetesCoreCachingAgent.

* perf(kubernetes): Fix all agents handling on-demand requests

Currently sending an on-demand cache update for a resource that
is not namespaced will cause all caching agents to try to handle
the update. This can cause a very high load on redis if there are
a lot of agents (as might be expected in a large installation of
Spinnaker).

Designate one agent to handle requests that are not namespaced
instead of having them handled by all agents.

* test(kubernetes): Improve KubernetesCoreCachingAgent tests

Instead of mocking the ProviderCache, use an in-memory cache, and
validate that the cache has the expected results after on-demand
requests have been processed.

clouddriver-kubernetes-v2/clouddriver-kubernetes-v2.gradle

@@ -9,6 +9,7 @@ dependencies {
   implementation project(":clouddriver-security")
 
   compileOnly "org.projectlombok:lombok"
+  testCompile "org.projectlombok:lombok"
   annotationProcessor "org.projectlombok:lombok"
   testAnnotationProcessor "org.projectlombok:lombok"
 
@@ -28,7 +29,9 @@ dependencies {
   testImplementation "cglib:cglib-nodep"
   testImplementation "org.assertj:assertj-core"
   testImplementation "org.junit.jupiter:junit-jupiter-api"
+  testImplementation "org.junit.jupiter:junit-jupiter-params"
   testImplementation "org.junit.platform:junit-platform-runner"
+  testImplementation "org.mockito:mockito-core"
   testImplementation "org.objenesis:objenesis"
   testImplementation "org.spockframework:spock-core"
   testImplementation "org.spockframework:spock-spring"

clouddriver-kubernetes-v2/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgent.java

@@ -200,4 +200,12 @@ protected ImmutableList<String> getNamespaces() {
         .filter(n -> agentCount == 1 || Math.abs(n.hashCode() % agentCount) == agentIndex)
         .collect(ImmutableList.toImmutableList());
   }
+
+  /**
+   * Should this caching agent be responsible for caching cluster-scoped resources (ie, those that
+   * do not live in a particular namespace)?
+   */
+  protected boolean handleClusterScopedResources() {
+    return agentIndex == 0;
+  }
 }

clouddriver-kubernetes-v2/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2OnDemandCachingAgent.java

@@ -311,7 +311,7 @@ public OnDemandAgent.OnDemandResult handle(ProviderCache providerCache, Map<Stri
       namespace = "";
     }
 
-    if (!StringUtils.isEmpty(namespace) && !getNamespaces().contains(namespace)) {
+    if (!handleNamespace(namespace)) {
       return null;
     }
 
@@ -399,4 +399,11 @@ private Map<String, String> mapKeyToOnDemandResult(Keys.InfrastructureCacheKey k
   private boolean handleReadRequests() {
     return agentIndex == 0;
   }
+
+  private boolean handleNamespace(String namespace) {
+    if (StringUtils.isEmpty(namespace)) {
+      return handleClusterScopedResources();
+    }
+    return getNamespaces().contains(namespace);
+  }
 }