feat(cf): Logs support for run job stage and app instance (#3900)

* Added the Logs (doppler) service for the Cloud Foundry Client

Implemented the `recentLogs` endpoint only.

* Ensuring the jobName is unique

* Implemented CloudFoundryInstanceProvider.getConsoleOutput() using Doppler's recent logs.
Supports retrieving task or app logs.

clouddriver-cloudfoundry/clouddriver-cloudfoundry.gradle

@@ -1,3 +1,25 @@
+apply plugin: 'com.google.protobuf'
+
+buildscript {
+  repositories {
+    jcenter()
+  }
+
+  dependencies {
+    classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.9'
+  }
+}
+
+ext {
+  protobufVersion = '3.6.1'
+}
+
+protobuf {
+  protoc {
+    artifact = "com.google.protobuf:protoc:$protobufVersion"
+  }
+}
+
 test {
   useJUnitPlatform()
 }
@@ -37,6 +59,9 @@ dependencies {
   implementation "org.springframework.boot:spring-boot-starter-web"
   implementation "org.springframework.cloud:spring-cloud-context"
   implementation "org.yaml:snakeyaml:1.24"
+  implementation "com.google.protobuf:protobuf-java:$protobufVersion"
+  implementation "com.google.protobuf:protobuf-java-util:$protobufVersion"
+  implementation "commons-fileupload:commons-fileupload:1.4"
 
   testImplementation "org.assertj:assertj-core"
   testImplementation "org.junit.jupiter:junit-jupiter-api"

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/CloudFoundryClient.java

@@ -32,4 +32,6 @@ public interface CloudFoundryClient {
   ServiceKeys getServiceKeys();
 
   Tasks getTasks();
+
+  Logs getLogs();
 }

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/HttpCloudFoundryClient.java

@@ -25,6 +25,7 @@
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.AuthenticationService;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.ConfigService;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.DomainService;
+import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.DopplerService;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.OrganizationService;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.RouteService;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.ServiceInstanceService;
@@ -38,26 +39,43 @@
 import io.github.resilience4j.retry.IntervalFunction;
 import io.github.resilience4j.retry.Retry;
 import io.github.resilience4j.retry.RetryConfig;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.lang.reflect.Type;
 import java.net.SocketTimeoutException;
 import java.nio.charset.Charset;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
 import lombok.extern.slf4j.Slf4j;
 import okio.Buffer;
 import okio.BufferedSource;
+import org.apache.commons.fileupload.MultipartStream;
+import org.cloudfoundry.dropsonde.events.EventFactory.Envelope;
 import retrofit.RequestInterceptor;
 import retrofit.RestAdapter;
 import retrofit.client.OkClient;
+import retrofit.converter.ConversionException;
+import retrofit.converter.Converter;
 import retrofit.converter.JacksonConverter;
+import retrofit.mime.TypedInput;
+import retrofit.mime.TypedOutput;
 
+/**
+ * Waiting for this issue to be resolved before replacing this class by the CF Java Client:
+ * https://github.com/cloudfoundry/cf-java-client/issues/938
+ */
 @Slf4j
 public class HttpCloudFoundryClient implements CloudFoundryClient {
   private final String apiHost;
@@ -79,6 +97,7 @@ public class HttpCloudFoundryClient implements CloudFoundryClient {
   private ServiceInstances serviceInstances;
   private ServiceKeys serviceKeys;
   private Tasks tasks;
+  private Logs logs;
 
   private final RequestInterceptor oauthInterceptor =
       new RequestInterceptor() {
@@ -203,6 +222,16 @@ public HttpCloudFoundryClient(
         new Routes(account, createService(RouteService.class), applications, domains, spaces);
     this.serviceKeys = new ServiceKeys(createService(ServiceKeyService.class), spaces);
     this.tasks = new Tasks(createService(TaskService.class));
+
+    this.logs =
+        new Logs(
+            new RestAdapter.Builder()
+                .setEndpoint("https://" + apiHost.replaceAll("^api\\.", "doppler."))
+                .setClient(new OkClient(okHttpClient))
+                .setConverter(new ProtobufDopplerEnvelopeConverter())
+                .setRequestInterceptor(oauthInterceptor)
+                .build()
+                .create(DopplerService.class));
   }
 
   private static OkHttpClient createHttpClient(boolean skipSslValidation) {
@@ -309,4 +338,57 @@ public ServiceKeys getServiceKeys() {
   public Tasks getTasks() {
     return tasks;
   }
+
+  @Override
+  public Logs getLogs() {
+    return logs;
+  }
+
+  static class ProtobufDopplerEnvelopeConverter implements Converter {
+    @Override
+    public Object fromBody(TypedInput body, Type type) throws ConversionException {
+      try {
+        byte[] boundaryBytes = extractMultipartBoundary(body.mimeType()).getBytes();
+        MultipartStream multipartStream = new MultipartStream(body.in(), boundaryBytes, 4096, null);
+
+        List<Envelope> envelopes = new ArrayList<>();
+        ByteArrayOutputStream os = new ByteArrayOutputStream(4096);
+
+        boolean nextPart = multipartStream.skipPreamble();
+        while (nextPart) {
+          // Skipping the empty part headers
+          multipartStream.readByte(); // 0x0D
+          multipartStream.readByte(); // 0x0A
+
+          os.reset();
+          multipartStream.readBodyData(os);
+          envelopes.add(Envelope.parseFrom(os.toByteArray()));
+
+          nextPart = multipartStream.readBoundary();
+        }
+
+        return envelopes;
+      } catch (IOException e) {
+        throw new ConversionException(e);
+      }
+    }
+
+    @Override
+    public TypedOutput toBody(Object object) {
+      throw new UnsupportedOperationException("Deserializer only");
+    }
+
+    private static Pattern BOUNDARY_PATTERN = Pattern.compile("multipart/.+; boundary=(.*)");
+
+    private static String extractMultipartBoundary(String contentType) {
+      Matcher matcher = BOUNDARY_PATTERN.matcher(contentType);
+      if (matcher.matches()) {
+        return matcher.group(1);
+      } else {
+        throw new IllegalStateException(
+            String.format(
+                "Content-Type %s does not contain a valid multipart boundary", contentType));
+      }
+    }
+  }
 }

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/Logs.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.cloudfoundry.client;
+
+import static com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClientUtils.safelyCall;
+import static java.util.stream.Collectors.joining;
+
+import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.DopplerService;
+import java.util.Comparator;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.cloudfoundry.dropsonde.events.EventFactory.Envelope;
+import org.cloudfoundry.dropsonde.events.EventFactory.Envelope.EventType;
+import org.cloudfoundry.dropsonde.events.LogFactory.LogMessage;
+
+@RequiredArgsConstructor
+public class Logs {
+  private final DopplerService api;
+
+  public String recentApplicationLogs(String applicationGuid, int instanceIndex) {
+    return recentLogsFiltered(applicationGuid, "APP/PROC/WEB", instanceIndex);
+  }
+
+  public String recentTaskLogs(String applicationGuid, String taskName) {
+    return recentLogsFiltered(applicationGuid, "APP/TASK/" + taskName, 0);
+  }
+
+  public List<Envelope> recentLogs(String applicationGuid) {
+    return safelyCall(() -> api.recentLogs(applicationGuid))
+        .orElseThrow(IllegalStateException::new);
+  }
+
+  private String recentLogsFiltered(
+      String applicationGuid, String logSourceFilter, int instanceIndex) {
+    List<Envelope> envelopes = recentLogs(applicationGuid);
+
+    return envelopes.stream()
+        .filter(e -> e.getEventType().equals(EventType.LogMessage))
+        .map(Envelope::getLogMessage)
+        .filter(
+            logMessage ->
+                logSourceFilter.equals(logMessage.getSourceType())
+                    && logMessage.getSourceInstance().equals(String.valueOf(instanceIndex)))
+        .sorted(Comparator.comparingLong(LogMessage::getTimestamp))
+        .map(msg -> msg.getMessage().toStringUtf8())
+        .collect(joining("\n"));
+  }
+}

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/DopplerService.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.cloudfoundry.client.api;
+
+import java.util.List;
+import org.cloudfoundry.dropsonde.events.EventFactory.Envelope;
+import retrofit.http.GET;
+import retrofit.http.Path;
+
+public interface DopplerService {
+  @GET("/apps/{guid}/recentlogs")
+  List<Envelope> recentLogs(@Path("guid") String appGuid);
+}

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v3/Task.java

@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
 import java.time.ZonedDateTime;
+import java.util.Map;
 import lombok.Builder;
 import lombok.Value;
 
@@ -31,6 +32,7 @@ public class Task {
   private State state;
   private ZonedDateTime createdAt;
   private ZonedDateTime updatedAt;
+  private Map<String, Link> links;
 
   public enum State {
     SUCCEEDED,

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/job/CloudFoundryRunJobOperation.java

@@ -25,6 +25,7 @@
 import java.util.Collections;
 import java.util.List;
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
 
 @RequiredArgsConstructor
 public class CloudFoundryRunJobOperation implements AtomicOperation<DeploymentResult> {
@@ -34,11 +35,16 @@ public class CloudFoundryRunJobOperation implements AtomicOperation<DeploymentRe
   @Override
   public DeploymentResult operate(List priorOutputs) {
     CloudFoundryClient client = description.getClient();
-    String jobName = description.getJobName();
     CloudFoundryServerGroup serverGroup = description.getServerGroup();
     String applicationGuid = serverGroup.getId();
     String applicationName = serverGroup.getName();
 
+    // make the job name unique by appending to it a random string so its logs are filterable
+    String originalName = description.getJobName();
+    String randomString = Long.toHexString(Double.doubleToLongBits(Math.random()));
+    String jobName =
+        (StringUtils.isNotEmpty(originalName) ? originalName + "-" : "") + randomString;
+
     TaskRepository.threadLocalTask
         .get()
         .updateStatus(

clouddriver-cloudfoundry/src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/job/CloudFoundryRunJobOperationDescription.java

@@ -18,6 +18,7 @@
 
 import com.netflix.spinnaker.clouddriver.cloudfoundry.deploy.description.AbstractCloudFoundryDescription;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.model.CloudFoundryServerGroup;
+import javax.annotation.Nullable;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
@@ -26,6 +27,6 @@
 public class CloudFoundryRunJobOperationDescription extends AbstractCloudFoundryDescription {
 
   private CloudFoundryServerGroup serverGroup;
-  private String jobName;
+  @Nullable private String jobName;
   private String command;
 }