feat(plugins): Track plugin releases by plugin id (#757)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/PluginCache.java

@@ -16,48 +16,33 @@
  */
 package com.netflix.spinnaker.igor.plugins;
 
-import com.netflix.spinnaker.igor.IgorConfigurationProperties;
-import com.netflix.spinnaker.kork.jedis.RedisClientDelegate;
 import java.time.Instant;
-import jline.internal.Nullable;
-
-/** TODO(rz): Currently only supports front50 as a repository. */
-public class PluginCache {
-
-  private static final String ID = "plugins";
-  private static final String FRONT50_REPOSITORY = "front50";
-
-  private final RedisClientDelegate redisClientDelegate;
-  private final IgorConfigurationProperties igorConfigurationProperties;
-
-  public PluginCache(
-      RedisClientDelegate redisClientDelegate,
-      IgorConfigurationProperties igorConfigurationProperties) {
-    this.redisClientDelegate = redisClientDelegate;
-    this.igorConfigurationProperties = igorConfigurationProperties;
-  }
-
-  public void setLastPollCycleTimestamp(Instant timestamp) {
-    redisClientDelegate.withCommandsClient(
-        c -> {
-          c.hset(key(), FRONT50_REPOSITORY, String.valueOf(timestamp.toEpochMilli()));
-        });
-  }
-
+import java.util.Map;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/** Responsible for tracking seen plugin releases. */
+public interface PluginCache {
+
+  /**
+   * Sets the last time the poller updated for the provided pluginId.
+   *
+   * <p>This should only be called when a new plugin release has been discovered, using the release
+   * date of the plugin release as the timestamp.
+   */
+  void setLastPollCycle(@Nonnull String pluginId, @Nonnull Instant timestamp);
+
+  /**
+   * Get the last time the poller indexed the given pluginId. It will return null if no cache record
+   * exists.
+   *
+   * @param pluginId
+   * @return
+   */
   @Nullable
-  public Instant getLastPollCycleTimestamp() {
-    return redisClientDelegate.withCommandsClient(
-        c -> {
-          String timestamp = c.hget(key(), FRONT50_REPOSITORY);
-          if (timestamp == null) {
-            return null;
-          } else {
-            return Instant.ofEpochMilli(Long.parseLong(timestamp));
-          }
-        });
-  }
+  Instant getLastPollCycle(@Nonnull String pluginId);
 
-  private String key() {
-    return igorConfigurationProperties.getSpinnaker().getJedis().getPrefix() + ":" + ID;
-  }
+  /** List all latest poll cycle timestamps, indexed by plugin ID. */
+  @Nonnull
+  Map<String, Instant> listLastPollCycles();
 }

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/PluginsBuildMonitor.java

@@ -59,27 +59,39 @@ public PluginsBuildMonitor(
   @Override
   protected PluginPollingDelta generateDelta(PollContext ctx) {
     return new PluginPollingDelta(
-        pluginInfoService.getPluginReleasesSince(cache.getLastPollCycleTimestamp()).stream()
+        pluginInfoService.getPluginReleasesSinceTimestamps(cache.listLastPollCycles()).stream()
             .map(PluginDelta::new)
             .collect(Collectors.toList()));
   }
 
   @Override
   protected void commitDelta(PluginPollingDelta delta, boolean sendEvents) {
     log.info("Found {} new plugin releases", delta.items.size());
-    delta.items.forEach(
-        item -> {
-          if (sendEvents) {
-            postEvent(item.pluginRelease);
-          } else {
-            log.debug("{} processed, but not sending event", item.pluginRelease);
-          }
-        });
 
+    // Group the items by their plugin ID, submitting each release (even if there are more than one
+    // per plugin ID).
+    // After submitting the events, the most recent plugin release date for each plugin ID is then
+    // used to update the
+    // cache's last poll cycle value.
     delta.items.stream()
-        .map(it -> Instant.parse(it.pluginRelease.getReleaseDate()))
-        .max(Comparator.naturalOrder())
-        .ifPresent(cache::setLastPollCycleTimestamp);
+        .collect(Collectors.groupingBy(d -> d.pluginRelease.getPluginId()))
+        .forEach(
+            (pluginId, pluginDeltas) -> {
+              pluginDeltas.forEach(
+                  item -> {
+                    if (sendEvents) {
+                      postEvent(item.pluginRelease);
+                    } else {
+                      log.debug("{} processed, but not sending event", item.pluginRelease);
+                    }
+                  });
+
+              pluginDeltas.stream()
+                  // Already validated this release is going to be valid, so not error checking.
+                  .map(it -> Instant.parse(it.pluginRelease.getReleaseDate()))
+                  .max(Comparator.naturalOrder())
+                  .ifPresent(ts -> cache.setLastPollCycle(pluginId, ts));
+            });
   }
 
   private void postEvent(PluginRelease release) {

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/RedisPluginCache.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins;
+
+import com.netflix.spinnaker.igor.IgorConfigurationProperties;
+import com.netflix.spinnaker.kork.jedis.RedisClientDelegate;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import javax.annotation.Nonnull;
+import jline.internal.Nullable;
+
+public class RedisPluginCache implements PluginCache {
+
+  private static final String ID = "plugins";
+
+  private final RedisClientDelegate redisClientDelegate;
+  private final IgorConfigurationProperties igorConfigurationProperties;
+
+  public RedisPluginCache(
+      RedisClientDelegate redisClientDelegate,
+      IgorConfigurationProperties igorConfigurationProperties) {
+    this.redisClientDelegate = redisClientDelegate;
+    this.igorConfigurationProperties = igorConfigurationProperties;
+  }
+
+  private String key() {
+    return igorConfigurationProperties.getSpinnaker().getJedis().getPrefix() + ":" + ID;
+  }
+
+  @Override
+  public void setLastPollCycle(@Nonnull String pluginId, @Nonnull Instant timestamp) {
+    redisClientDelegate.withCommandsClient(
+        c -> {
+          c.hset(key(), pluginId, String.valueOf(timestamp.toEpochMilli()));
+        });
+  }
+
+  @Override
+  @Nullable
+  public Instant getLastPollCycle(@Nonnull String pluginId) {
+    return redisClientDelegate.withCommandsClient(
+        c -> {
+          return Optional.ofNullable(c.hget(key(), pluginId))
+              .map(Long::parseLong)
+              .map(Instant::ofEpochMilli)
+              .orElse(null);
+        });
+  }
+
+  @Override
+  @Nonnull
+  public Map<String, Instant> listLastPollCycles() {
+    return redisClientDelegate.withCommandsClient(
+        c -> {
+          Map<String, Instant> cycles = new HashMap<>();
+          c.hgetAll(key())
+              .forEach(
+                  (pluginId, ts) -> cycles.put(pluginId, Instant.ofEpochMilli(Long.parseLong(ts))));
+          return cycles;
+        });
+  }
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/front50/PluginInfo.java

@@ -33,6 +33,7 @@ static class Release {
     final String date;
     final String requires;
     final String url;
+    final String sha512sum;
     final boolean preferred;
     final String lastModified;
 
@@ -41,12 +42,14 @@ public Release(
         String date,
         String requires,
         String url,
+        String sha512sum,
         boolean preferred,
         String lastModified) {
       this.version = version;
       this.date = date;
       this.requires = requires;
       this.url = url;
+      this.sha512sum = sha512sum;
       this.preferred = preferred;
       this.lastModified = lastModified;
     }

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/front50/PluginReleaseService.java

@@ -20,8 +20,9 @@
 import com.netflix.spinnaker.security.AuthenticatedRequest;
 import java.time.Instant;
 import java.time.format.DateTimeParseException;
-import java.util.List;
+import java.util.*;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,27 +36,33 @@ public PluginReleaseService(Front50Service front50Service) {
     this.front50Service = front50Service;
   }
 
-  public List<PluginRelease> getPluginReleasesSince(Instant timestamp) {
-    if (timestamp == null) {
-      return getPluginReleases();
-    }
-
+  public List<PluginRelease> getPluginReleasesSinceTimestamps(
+      @Nonnull Map<String, Instant> pluginTimestamps) {
     return getPluginReleases().stream()
         .filter(
-            r -> {
-              try {
-                return Instant.parse(r.getReleaseDate()).isAfter(timestamp);
-              } catch (DateTimeParseException e) {
-                log.error(
-                    "Failed parsing plugin timestamp for '{}': '{}', cannot index plugin",
-                    r.getPluginId(),
-                    r.getReleaseDate());
-                return false;
-              }
+            release -> {
+              Instant lastCycle =
+                  Optional.ofNullable(pluginTimestamps.get(release.getPluginId()))
+                      .orElse(Instant.EPOCH);
+              return parseReleaseTimestamp(release)
+                  .map(releaseTs -> releaseTs.isAfter(lastCycle))
+                  .orElse(false);
             })
         .collect(Collectors.toList());
   }
 
+  private Optional<Instant> parseReleaseTimestamp(PluginRelease release) {
+    try {
+      return Optional.of(Instant.parse(release.getReleaseDate()));
+    } catch (DateTimeParseException e) {
+      log.error(
+          "Failed parsing plugin timestamp for '{}': '{}', cannot index plugin",
+          release.getPluginId(),
+          release.getReleaseDate());
+      return Optional.empty();
+    }
+  }
+
   private List<PluginRelease> getPluginReleases() {
     return AuthenticatedRequest.allowAnonymous(
         () ->
@@ -69,8 +76,10 @@ private List<PluginRelease> getPluginReleases() {
                                         info.id,
                                         release.version,
                                         release.date,
+                                        release.requires,
                                         PluginRequiresParser.parseRequires(release.requires),
                                         release.url,
+                                        release.sha512sum,
                                         release.preferred,
                                         release.lastModified)))
                 .collect(Collectors.toList()));

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/model/PluginRelease.java

@@ -22,24 +22,30 @@ public class PluginRelease {
   private final String pluginId;
   private final String version;
   private final String releaseDate;
-  private final List<ServiceRequirement> requires;
+  private final String requires;
+  private final List<ServiceRequirement> parsedRequires;
   private final String binaryUrl;
+  private final String sha512sum;
   private final boolean preferred;
   private final String lastModified;
 
   public PluginRelease(
       String pluginId,
       String version,
       String releaseDate,
-      List<ServiceRequirement> requires,
+      String requires,
+      List<ServiceRequirement> parsedRequires,
       String binaryUrl,
+      String sha512sum,
       boolean preferred,
       String lastModified) {
     this.pluginId = pluginId;
     this.version = version;
     this.releaseDate = releaseDate;
     this.requires = requires;
+    this.parsedRequires = parsedRequires;
     this.binaryUrl = binaryUrl;
+    this.sha512sum = sha512sum;
     this.preferred = preferred;
     this.lastModified = lastModified;
   }
@@ -56,14 +62,22 @@ public String getReleaseDate() {
     return releaseDate;
   }
 
-  public List<ServiceRequirement> getRequires() {
+  public String getRequires() {
     return requires;
   }
 
+  public List<ServiceRequirement> getParsedRequires() {
+    return parsedRequires;
+  }
+
   public String getBinaryUrl() {
     return binaryUrl;
   }
 
+  public String getSha512sum() {
+    return sha512sum;
+  }
+
   public boolean isPreferred() {
     return preferred;
   }