[Transform] add a health section to transform stats (#90760)

adds a health section to the transform stats endpoint and implements reporting assignment, indexing/search and persistence problems, together with a overall health state.

docs/changelog/90760.yaml

@@ -0,0 +1,5 @@
+pr: 90760
+summary: Add a health section to transform stats
+area: Transform
+type: enhancement
+issues: []

docs/reference/transform/apis/get-transform-stats.asciidoc

@@ -142,6 +142,53 @@ that the {transform} is failing to keep up.
 ====
 //End checkpointing
 
+//Begin health
+`health`::
+(object) Health indicator for this {transform}.
++
+.Properties of `health`
+[%collapsible%open]
+====
+`status`::
+    (string) Health status of this transform. Statuses are:
+
+    `green`:::
+    The transform is healthy.
+
+    `unknown`:::
+    The health of the transform could not be determined.
+
+    `yellow`:::
+    The functionality of the transform is in a degraded state and may need remediation
+    to avoid the health becoming `red`.
+
+    `red`:::
+    The transform is experiencing an outage or is unavailable for use.
+
+`issues`::
+    (Optional, array) If a non-healthy status is returned, contains a list of issues
+    of the transform.
++
+.Properties of `issues`
+[%collapsible%open]
+========
+`issue`::
+    (string) A description of the issue.
+
+`details`::
+    (Optional, string) Details about the issue.
+
+`count`::
+    (integer) Number of times the issue has occured since it started.
+
+`first_occurrence`::
+    (Optional, date) The timestamp this issue occured for the first time.
+========
+//End issues
+
+====
+//End health
+
 `id`::
 (string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id]
@@ -328,6 +375,9 @@ The API returns the following results:
           "time_upper_bound_millis" : 1585344498220
         },
         "changes_last_detected_at" : 1585344558219
+      },
+      "health": {
+        "status": "green"
       }
     }
   ]

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealth.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.transforms;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.health.HealthStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class TransformHealth implements Writeable, ToXContentObject {
+
+    public static final TransformHealth GREEN = new TransformHealth(HealthStatus.GREEN, null);
+
+    public static final TransformHealth UNKNOWN = new TransformHealth(HealthStatus.UNKNOWN, null);
+
+    private static final String STATUS = "status";
+    private static final String ISSUES = "issues";
+
+    private final HealthStatus status;
+    private final List<TransformHealthIssue> issues;
+
+    public TransformHealth(HealthStatus status, List<TransformHealthIssue> issues) {
+        this.status = status;
+        this.issues = issues;
+    }
+
+    public TransformHealth(StreamInput in) throws IOException {
+        this.status = in.readEnum(HealthStatus.class);
+        this.issues = in.readOptionalList(TransformHealthIssue::new);
+    }
+
+    public HealthStatus getStatus() {
+        return status;
+    }
+
+    public List<TransformHealthIssue> getIssues() {
+        return issues;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(STATUS, status.xContentValue());
+        if (issues != null && issues.isEmpty() == false) {
+            builder.field(ISSUES, issues);
+        }
+        return builder.endObject();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        status.writeTo(out);
+        out.writeOptionalCollection(issues);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        TransformHealth that = (TransformHealth) other;
+
+        return this.status.value() == that.status.value() && Objects.equals(this.issues, that.issues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(status, issues);
+    }
+
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssue.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.transforms;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Objects;
+
+public class TransformHealthIssue implements Writeable, ToXContentObject {
+
+    private static final String ISSUE = "issue";
+    private static final String DETAILS = "details";
+    private static final String COUNT = "count";
+    private static final String FIRST_OCCURRENCE = "first_occurrence";
+    private static final String FIRST_OCCURRENCE_HUMAN_READABLE = FIRST_OCCURRENCE + "_string";
+
+    private final String issue;
+    private final String details;
+    private final int count;
+    private final Instant firstOccurrence;
+
+    public TransformHealthIssue(String issue, String details, int count, Instant firstOccurrence) {
+        this.issue = Objects.requireNonNull(issue);
+        this.details = details;
+        if (count < 1) {
+            throw new IllegalArgumentException("[count] must be at least 1, got: " + count);
+        }
+        this.count = count;
+        this.firstOccurrence = firstOccurrence != null ? firstOccurrence.truncatedTo(ChronoUnit.MILLIS) : null;
+    }
+
+    public TransformHealthIssue(StreamInput in) throws IOException {
+        this.issue = in.readString();
+        this.details = in.readOptionalString();
+        this.count = in.readVInt();
+        this.firstOccurrence = in.readOptionalInstant();
+    }
+
+    public String getIssue() {
+        return issue;
+    }
+
+    public String getDetails() {
+        return details;
+    }
+
+    public int getCount() {
+        return count;
+    }
+
+    public Instant getFirstOccurrence() {
+        return firstOccurrence;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ISSUE, issue);
+        if (Strings.isNullOrEmpty(details) == false) {
+            builder.field(DETAILS, details);
+        }
+        builder.field(COUNT, count);
+        if (firstOccurrence != null) {
+            builder.timeField(FIRST_OCCURRENCE, FIRST_OCCURRENCE_HUMAN_READABLE, firstOccurrence.toEpochMilli());
+        }
+        return builder.endObject();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(issue);
+        out.writeOptionalString(details);
+        out.writeVInt(count);
+        out.writeOptionalInstant(firstOccurrence);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        TransformHealthIssue that = (TransformHealthIssue) other;
+
+        return this.count == that.count
+            && Objects.equals(this.issue, that.issue)
+            && Objects.equals(this.details, that.details)
+            && Objects.equals(this.firstOccurrence, that.firstOccurrence);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(issue, details, count, firstOccurrence);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}