feat(plugins): Add new front50 plugin info monitor (#723)

Allows Spinnaker to trigger pipelines off of new plugin releases into front50.

igor-core/src/main/java/com/netflix/spinnaker/igor/IgorConfigurationProperties.java

@@ -156,6 +156,8 @@ public static class ServicesProperties {
     /** Config for keel connectivity. */
     @NestedConfigurationProperty private ServiceConfiguration keel = new ServiceConfiguration();
 
+    @NestedConfigurationProperty private ServiceConfiguration front50 = new ServiceConfiguration();
+
     @Data
     public static class ServiceConfiguration {
       /**

igor-monitor-plugins/igor-monitor-plugins.gradle

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+dependencies {
+  implementation project(":igor-core")
+
+  implementation "com.netflix.spinnaker.kork:kork-artifacts"
+  implementation "com.netflix.spinnaker.kork:kork-core"
+  implementation "com.netflix.spinnaker.kork:kork-jedis"
+  implementation "com.netflix.spinnaker.kork:kork-security"
+
+  implementation "org.springframework.boot:spring-boot-starter-actuator"
+
+  implementation "com.squareup.retrofit:retrofit"
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/PluginCache.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins;
+
+import com.netflix.spinnaker.igor.IgorConfigurationProperties;
+import com.netflix.spinnaker.kork.jedis.RedisClientDelegate;
+import java.time.Instant;
+import jline.internal.Nullable;
+
+/** TODO(rz): Currently only supports front50 as a repository. */
+public class PluginCache {
+
+  private static final String ID = "plugins";
+  private static final String FRONT50_REPOSITORY = "front50";
+
+  private final RedisClientDelegate redisClientDelegate;
+  private final IgorConfigurationProperties igorConfigurationProperties;
+
+  public PluginCache(
+      RedisClientDelegate redisClientDelegate,
+      IgorConfigurationProperties igorConfigurationProperties) {
+    this.redisClientDelegate = redisClientDelegate;
+    this.igorConfigurationProperties = igorConfigurationProperties;
+  }
+
+  public void setLastPollCycleTimestamp(Instant timestamp) {
+    redisClientDelegate.withCommandsClient(
+        c -> {
+          c.hset(key(), FRONT50_REPOSITORY, String.valueOf(timestamp.toEpochMilli()));
+        });
+  }
+
+  @Nullable
+  public Instant getLastPollCycleTimestamp() {
+    return redisClientDelegate.withCommandsClient(
+        c -> {
+          String timestamp = c.hget(key(), FRONT50_REPOSITORY);
+          if (timestamp == null) {
+            return null;
+          } else {
+            return Instant.ofEpochMilli(Long.parseLong(timestamp));
+          }
+        });
+  }
+
+  private String key() {
+    return igorConfigurationProperties.getSpinnaker().getJedis().getPrefix() + ":" + ID;
+  }
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/PluginsBuildMonitor.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins;
+
+import com.netflix.discovery.DiscoveryClient;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.igor.IgorConfigurationProperties;
+import com.netflix.spinnaker.igor.history.EchoService;
+import com.netflix.spinnaker.igor.plugins.front50.PluginReleaseService;
+import com.netflix.spinnaker.igor.plugins.model.PluginEvent;
+import com.netflix.spinnaker.igor.plugins.model.PluginRelease;
+import com.netflix.spinnaker.igor.polling.*;
+import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService;
+import com.netflix.spinnaker.security.AuthenticatedRequest;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.Data;
+
+public class PluginsBuildMonitor
+    extends CommonPollingMonitor<
+        PluginsBuildMonitor.PluginDelta, PluginsBuildMonitor.PluginPollingDelta> {
+
+  private final PluginReleaseService pluginInfoService;
+  private final PluginCache cache;
+  private final Optional<EchoService> echoService;
+
+  public PluginsBuildMonitor(
+      IgorConfigurationProperties igorProperties,
+      Registry registry,
+      DynamicConfigService dynamicConfigService,
+      Optional<DiscoveryClient> discoveryClient,
+      Optional<LockService> lockService,
+      PluginReleaseService pluginInfoService,
+      PluginCache cache,
+      Optional<EchoService> echoService) {
+    super(igorProperties, registry, dynamicConfigService, discoveryClient, lockService);
+    this.pluginInfoService = pluginInfoService;
+    this.cache = cache;
+    this.echoService = echoService;
+  }
+
+  @Override
+  protected PluginPollingDelta generateDelta(PollContext ctx) {
+    return new PluginPollingDelta(
+        pluginInfoService.getPluginReleasesSince(cache.getLastPollCycleTimestamp()).stream()
+            .map(PluginDelta::new)
+            .collect(Collectors.toList()));
+  }
+
+  @Override
+  protected void commitDelta(PluginPollingDelta delta, boolean sendEvents) {
+    delta.items.forEach(
+        item -> {
+          if (sendEvents) {
+            postEvent(item.pluginRelease);
+            log.debug("{} event posted", item.pluginRelease);
+          }
+        });
+
+    delta.items.stream()
+        .map(it -> Instant.parse(it.pluginRelease.getTimestamp()))
+        .max(Comparator.naturalOrder())
+        .ifPresent(cache::setLastPollCycleTimestamp);
+  }
+
+  private void postEvent(PluginRelease release) {
+    if (!echoService.isPresent()) {
+      log.warn("Cannot send new plugin notification: Echo is not configured");
+      registry.counter(
+          missedNotificationId.withTag("monitor", PluginsBuildMonitor.class.getSimpleName()));
+    } else if (release != null) {
+      AuthenticatedRequest.allowAnonymous(
+          () -> echoService.get().postEvent(new PluginEvent(release)));
+    }
+  }
+
+  @Override
+  public void poll(boolean sendEvents) {
+    pollSingle(new PollContext("front50"));
+  }
+
+  @Override
+  public String getName() {
+    return "pluginsMonitor";
+  }
+
+  @Data
+  static class PluginDelta implements DeltaItem {
+    private final PluginRelease pluginRelease;
+  }
+
+  @Data
+  static class PluginPollingDelta implements PollingDelta<PluginDelta> {
+    private final List<PluginDelta> items;
+  }
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/front50/Front50Service.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins.front50;
+
+import java.util.List;
+import retrofit.http.GET;
+
+public interface Front50Service {
+
+  @GET("/pluginInfo")
+  List<PluginInfo> listPluginInfo();
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/front50/PluginInfo.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins.front50;
+
+import java.util.List;
+
+public class PluginInfo {
+
+  final String id;
+  final List<Release> releases;
+
+  public PluginInfo(String id, List<Release> releases) {
+    this.id = id;
+    this.releases = releases;
+  }
+
+  static class Release {
+    final String version;
+    final String date;
+    final String requires;
+    final String url;
+    final boolean preferred;
+    final String lastModified;
+
+    public Release(
+        String version,
+        String date,
+        String requires,
+        String url,
+        boolean preferred,
+        String lastModified) {
+      this.version = version;
+      this.date = date;
+      this.requires = requires;
+      this.url = url;
+      this.preferred = preferred;
+      this.lastModified = lastModified;
+    }
+  }
+}

igor-monitor-plugins/src/main/java/com/netflix/spinnaker/igor/plugins/front50/PluginReleaseService.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2020 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.netflix.spinnaker.igor.plugins.front50;
+
+import com.netflix.spinnaker.igor.plugins.model.PluginRelease;
+import com.netflix.spinnaker.security.AuthenticatedRequest;
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PluginReleaseService {
+
+  private static final Logger log = LoggerFactory.getLogger(PluginReleaseService.class);
+
+  private final Front50Service front50Service;
+
+  public PluginReleaseService(Front50Service front50Service) {
+    this.front50Service = front50Service;
+  }
+
+  public List<PluginRelease> getPluginReleasesSince(Instant timestamp) {
+    if (timestamp == null) {
+      return getPluginReleases();
+    }
+
+    return getPluginReleases().stream()
+        .filter(
+            r -> {
+              try {
+                return Instant.parse(r.getTimestamp()).isAfter(timestamp);
+              } catch (DateTimeParseException e) {
+                log.error(
+                    "Failed parsing plugin timestamp for '{}': '{}', cannot index plugin",
+                    r.getPluginId(),
+                    getTimestamp(r));
+                return false;
+              }
+            })
+        .collect(Collectors.toList());
+  }
+
+  private List<PluginRelease> getPluginReleases() {
+    return AuthenticatedRequest.allowAnonymous(
+        () ->
+            front50Service.listPluginInfo().stream()
+                .flatMap(
+                    info ->
+                        info.releases.stream()
+                            .map(
+                                release ->
+                                    new PluginRelease(
+                                        info.id,
+                                        release.version,
+                                        release.date,
+                                        PluginRequiresParser.parseRequires(release.requires),
+                                        release.url,
+                                        release.preferred,
+                                        release.lastModified)))
+                .collect(Collectors.toList()));
+  }
+
+  private String getTimestamp(PluginRelease release) {
+    return Optional.ofNullable(release.getLastModified()).orElse(release.getReleaseDate());
+  }
+}