Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved subscription health problem indicators #972

Merged
merged 7 commits into from Jan 24, 2019
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -8,28 +8,27 @@
import java.util.Set;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Sets.immutableEnumSet;
import static java.util.Collections.emptySet;

public final class SubscriptionHealth {
public static final SubscriptionHealth HEALTHY = new SubscriptionHealth(Status.HEALTHY, emptySet());
public static final SubscriptionHealth NO_DATA = new SubscriptionHealth(Status.NO_DATA, emptySet());

private final Status status;
private final ImmutableSet<Problem> problems;
private final ImmutableSet<SubscriptionHealthProblem> problems;

@JsonCreator
private SubscriptionHealth(@JsonProperty("status") Status status,
@JsonProperty("problems") Set<Problem> problems) {
@JsonProperty("problems") Set<SubscriptionHealthProblem> problems) {
this.status = status;
this.problems = immutableEnumSet(problems);
this.problems = ImmutableSet.copyOf(problems);
}

public Status getStatus() {
return status;
}

public Set<Problem> getProblems() {
public Set<SubscriptionHealthProblem> getProblems() {
return problems;
}

Expand All @@ -55,7 +54,7 @@ public String toString() {
'}';
}

public static SubscriptionHealth of(Set<Problem> problems) {
public static SubscriptionHealth of(Set<SubscriptionHealthProblem> problems) {
checkNotNull(problems, "Set of health problems cannot be null");
if (problems.isEmpty()) {
return HEALTHY;
Expand All @@ -67,8 +66,4 @@ public static SubscriptionHealth of(Set<Problem> problems) {
public enum Status {
HEALTHY, UNHEALTHY, NO_DATA
}

public enum Problem {
LAGGING, SLOW, UNREACHABLE, TIMING_OUT, MALFUNCTIONING, RECEIVING_MALFORMED_MESSAGES
}
}
@@ -0,0 +1,96 @@
package pl.allegro.tech.hermes.api;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

import java.util.Objects;

import static java.lang.String.format;
import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.LAGGING;
import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.MALFUNCTIONING;
import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.RECEIVING_MALFORMED_MESSAGES;
import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.TIMING_OUT;
import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.UNREACHABLE;

public class SubscriptionHealthProblem {

public enum ProblemCode {
LAGGING, UNREACHABLE, TIMING_OUT, MALFUNCTIONING, RECEIVING_MALFORMED_MESSAGES
}

private final ProblemCode code;
private final String description;

@JsonCreator
private SubscriptionHealthProblem(@JsonProperty("code") ProblemCode code,
@JsonProperty("description") String description) {
this.code = code;
this.description = description;
}

public static SubscriptionHealthProblem lagging(long subscriptionLag) {
return new SubscriptionHealthProblem(
LAGGING,
format("Subscription lag is growing, current value is %d messages", subscriptionLag)
);
}

public static SubscriptionHealthProblem malfunctioning(double code5xxErrorsRate) {
return new SubscriptionHealthProblem(
MALFUNCTIONING,
format("Consuming service returns a lot of 5xx codes, currently %.0f 5xx/s", code5xxErrorsRate)
);
}

public static SubscriptionHealthProblem receivingMalformedMessages(double code4xxErrorsRate) {
return new SubscriptionHealthProblem(
RECEIVING_MALFORMED_MESSAGES,
format("Consuming service returns a lot of 4xx codes, currently %.0f 4xx/s", code4xxErrorsRate)
);
}

public static SubscriptionHealthProblem timingOut(double timeoutsRate) {
return new SubscriptionHealthProblem(
TIMING_OUT,
format("Consuming service times out a lot, currently %.0f timeouts/s", timeoutsRate)
);
}

public static SubscriptionHealthProblem unreachable(double otherErrorsRate) {
return new SubscriptionHealthProblem(
UNREACHABLE,
format("Unable to connect to consuming service instances, current rate is %.0f failures/s", otherErrorsRate)
);
}

public ProblemCode getCode() {
return code;
}

public String getDescription() {
return description;
}

@Override
public String toString() {
return code.name();
}

@Override
public int hashCode() {
return Objects.hash(code, description);
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
SubscriptionHealthProblem other = (SubscriptionHealthProblem) obj;
return Objects.equals(this.code, other.code)
&& Objects.equals(this.description, other.description);
}
}
Expand Up @@ -16,6 +16,7 @@ public class SubscriptionMetrics {
private Subscription.State state;
private String rate;
private String throughput;
private String batchRate;

private SubscriptionMetrics() {
}
Expand All @@ -26,7 +27,7 @@ public SubscriptionMetrics(@JsonProperty("delivered") long delivered, @JsonPrope
@JsonProperty("otherErrors") String otherErrors, @JsonProperty("codes2xx") String codes2xx,
@JsonProperty("codes4xx") String codes4xx, @JsonProperty("codes5xx") String codes5xx,
@JsonProperty("Subscription") Subscription.State state, @JsonProperty("rate") String rate,
@JsonProperty("throughput") String throughput) {
@JsonProperty("throughput") String throughput, @JsonProperty("batchRate") String batchRate) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Split into a separate lines please 😉

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

this.delivered = delivered;
this.discarded = discarded;
this.inflight = inflight;
Expand All @@ -38,6 +39,7 @@ public SubscriptionMetrics(@JsonProperty("delivered") long delivered, @JsonPrope
this.state = state;
this.rate = rate;
this.throughput = throughput;
this.batchRate = batchRate;
}

public long getDelivered() {
Expand Down Expand Up @@ -88,6 +90,10 @@ public String getThroughput() {
return throughput;
}

public String getBatchRate() {
return batchRate;
}

public static class Builder {
private SubscriptionMetrics subscriptionMetrics;

Expand Down Expand Up @@ -155,6 +161,11 @@ public Builder withThroughput(String throughput) {
return this;
}

public Builder withBatchRate(String batchRate) {
subscriptionMetrics.batchRate = batchRate;
return this;
}

public static Builder subscriptionMetrics() {
return new Builder();
}
Expand Down
Expand Up @@ -11,14 +11,14 @@ public class UnhealthySubscription {
private final String name;
private final String qualifiedTopicName;
private final MonitoringDetails.Severity severity;
private final Set<SubscriptionHealth.Problem> problems;
private final Set<SubscriptionHealthProblem> problems;

@JsonCreator
public UnhealthySubscription(
@JsonProperty("name") String name,
@JsonProperty("topicName") String qualifiedTopicName,
@JsonProperty("severity") MonitoringDetails.Severity severity,
@JsonProperty("problems") Set<SubscriptionHealth.Problem> problems) {
@JsonProperty("problems") Set<SubscriptionHealthProblem> problems) {
this.name = name;
this.qualifiedTopicName = qualifiedTopicName;
this.severity = severity;
Expand All @@ -45,7 +45,7 @@ public MonitoringDetails.Severity getSeverity() {
}

@JsonProperty("problems")
public Set<SubscriptionHealth.Problem> getProblems() {
public Set<SubscriptionHealthProblem> getProblems() {
return problems;
}

Expand Down
@@ -1,21 +1,21 @@
package pl.allegro.tech.hermes.api.endpoints;

import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import pl.allegro.tech.hermes.api.UnhealthySubscription;

import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import java.util.List;
import javax.ws.rs.core.Response;

import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import static javax.ws.rs.core.MediaType.TEXT_PLAIN;

@Path("unhealthy")
public interface UnhealthyEndpoint {

@GET
@Produces(APPLICATION_JSON)
@Produces({APPLICATION_JSON, TEXT_PLAIN})
@Path("/")
List<UnhealthySubscription> listUnhealthy(@QueryParam("ownerSourceName") String ownerSourceName,
@QueryParam("ownerId") String id,
@QueryParam("respectMonitoringSeverity") boolean respectMonitoringSeverity);
Response listUnhealthy(@QueryParam("ownerSourceName") String ownerSourceName,
@QueryParam("ownerId") String id,
@QueryParam("respectMonitoringSeverity") boolean respectMonitoringSeverity);
}
Expand Up @@ -8,15 +8,14 @@ health.factory('SubscriptionHealth', ['DiscoveryService', '$resource',
health: function (topicName, subscriptionName) {
return health.get({topicName: topicName, subscriptionName: subscriptionName})
.$promise.then(function (health) {
var problemOccurs = function (problem) {
return _.include(health.problems, problem)
var problemOccurs = function (problemCode) {
return _.some(health.problems, {code: problemCode})
};

return {
status: health.status,
problems: {
lagging: problemOccurs('LAGGING'),
slow: problemOccurs('SLOW'),
malfunctioning: problemOccurs('MALFUNCTIONING'),
receivingMalformedMessages: problemOccurs('RECEIVING_MALFORMED_MESSAGES'),
timingOut: problemOccurs('TIMING_OUT'),
Expand Down
4 changes: 0 additions & 4 deletions hermes-console/static/partials/subscription.html
Expand Up @@ -40,10 +40,6 @@ <h3 class="panel-title">Subscription health problems</h3>
<b>Subscription lag is growing</b>. Examine output rate and service response codes, looks like it is
not consuming at full speed.
</p>
<p ng-show="health.problems.slow">
<b>Consumption rate is lower than topic production rate</b>. Examine output rate and service response codes.
If everything is well, take a look at maximum rate, maybe it is too low?
</p>
<p ng-show="health.problems.malfunctioning">
<b>Consuming service returns a lot of 5xx codes</b>. Looks like it might be malfunctioning or doesn't know
how to handle messages. Take a look at "Last undelivered message" for more information.
Expand Down
@@ -1,11 +1,8 @@
package pl.allegro.tech.hermes.management.api;

import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import org.springframework.beans.factory.annotation.Autowired;
import pl.allegro.tech.hermes.api.OwnerId;
import pl.allegro.tech.hermes.api.UnhealthySubscription;
import pl.allegro.tech.hermes.management.domain.owner.OwnerSource;
import pl.allegro.tech.hermes.management.domain.owner.OwnerSourceNotFound;
import pl.allegro.tech.hermes.management.domain.owner.OwnerSources;
import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService;

Expand All @@ -14,9 +11,13 @@
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.GenericEntity;
import javax.ws.rs.core.Response;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;

import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import static javax.ws.rs.core.MediaType.TEXT_PLAIN;

@Path("unhealthy")
public class UnhealthyEndpoint {
Expand All @@ -32,17 +33,20 @@ public UnhealthyEndpoint(OwnerSources ownerSources,
}

@GET
@Produces(APPLICATION_JSON)
@Produces({APPLICATION_JSON, TEXT_PLAIN})
@Path("/")
public List<UnhealthySubscription> listUnhealthy(
public Response listUnhealthy(
@QueryParam("ownerSourceName") String ownerSourceName,
@QueryParam("ownerId") String id,
@DefaultValue("true") @QueryParam("respectMonitoringSeverity") boolean respectMonitoringSeverity) {

Optional<OwnerId> ownerId = resolveOwnerId(ownerSourceName, id);
return ownerId.isPresent()
List<UnhealthySubscription> unhealthySubscriptions = ownerId.isPresent()
? subscriptionService.getUnhealthyForOwner(ownerId.get(), respectMonitoringSeverity)
: subscriptionService.getAllUnhealthy(respectMonitoringSeverity);
return Response.ok()
.entity(new GenericEntity<List<UnhealthySubscription>>(unhealthySubscriptions){})
.build();
}

private Optional<OwnerId> resolveOwnerId(String ownerSourceName, String id) {
Expand Down
@@ -0,0 +1,65 @@
package pl.allegro.tech.hermes.management.api.writer;

import pl.allegro.tech.hermes.api.SubscriptionHealthProblem;
import pl.allegro.tech.hermes.api.UnhealthySubscription;

import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyWriter;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.List;

import static java.util.stream.Collectors.joining;
import static javax.ws.rs.core.MediaType.TEXT_PLAIN;

@Provider
@Produces(TEXT_PLAIN)
public class UnhealthySubscriptionListPlainTextBodyWriter implements MessageBodyWriter<List<UnhealthySubscription>> {

@Override
public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
if (List.class.isAssignableFrom(type) && genericType instanceof ParameterizedType) {
ParameterizedType parameterizedType = (ParameterizedType) genericType;
Type[] actualTypeArgs = parameterizedType.getActualTypeArguments();
return actualTypeArgs.length == 1 && actualTypeArgs[0].equals(UnhealthySubscription.class);
}
return false;
}

@Override
public long getSize(List<UnhealthySubscription> unhealthySubscriptionList,
Class<?> type,
Type genericType,
Annotation[] annotations,
MediaType mediaType) {
return -1; // According to JAX-RS 2.0 spec this method is deprecated and should return -1
}

@Override
public void writeTo(List<UnhealthySubscription> unhealthySubscriptionList,
Class<?> type,
Type genericType,
Annotation[] annotations,
MediaType mediaType,
MultivaluedMap<String, Object> httpHeaders,
OutputStream entityStream) throws IOException, WebApplicationException {
String body = unhealthySubscriptionList.stream()
.map(UnhealthySubscriptionListPlainTextBodyWriter::toPlainText)
.collect(joining("\r\n"));
entityStream.write(body.getBytes());
}

private static String toPlainText(UnhealthySubscription unhealthySubscription) {
String problemDescriptions = unhealthySubscription.getProblems().stream()
.map(SubscriptionHealthProblem::getDescription)
.collect(joining("; "));
return unhealthySubscription.getName() + " - " + problemDescriptions;
}
}