Merge pull request #596 from allegro/develop

prepare 0.9.0 release

CHANGELOG.md

@@ -1,3 +1,46 @@
+## 0.9.0 (17.10.2016)
+
+This release introduces Kafka 0.10 producer/consumer API and is no longer compatible with Kafka 0.8.x and 0.9.x deployments.
+
+### Features
+
+#### ([#558](https://github.com/allegro/hermes/issues/558)) Use Kafka 0.10 producer/consumer API
+
+**This change breaks backwards compatibility - Hermes will not run on 0.8.x, 0.9.x Kafka clusters**
+
+Hermes uses Kafka 0.10 APIs. The change is not big for producers in Frontend module, but it rearranged whole Consumers module.
+
+The benefits of moving to Kafka 0.10 (except from leaving the deprecated APIs behind) are:
+
+* decreased number of active threads: in cluster with ~600 subscriptions number of threads decreased from ~4400 to ~700
+* decreased memory consumption: same cluster, memory usage dropped by 10-20%
+* decreased CPU consumption: same cluster, day-to-day CPU consumption dropped by ~10%
+* greatly decreased shutdown time
+
+The change is transparent for the end users.
+
+**Upgrading note**
+
+Before upgraing, make sure that offsets are committed and stored in Kafka (option: `kafka.consumer.dual.commit.enabled` is set to `true` or 
+`kafka.consumer.offsets.storage` is set to `kafka` (default) in Consumers module).
+
+When upgrading, all Consumers should be stopped at once and started with new version.
+
+#### ([593](https://github.com/allegro/hermes/pull/593)) Confluent Schema Registry integration
+
+**Breaking change: Support for storing and validating JSON schemas has been removed**
+
+Hermes be integrated with [Confluent Schema Registry](https://github.com/confluentinc/schema-registry) to store and read Avro schemas. We kept existing integration with [schemarepo.org](http://schemarepo.org) repository. To switch between implementations, use `schema.repository.type` option:
+
+* `schema_repo` for "old" schemarepo.org
+* `schema_registry` for Confluent Schema Registry
+
+### Enhancements
+
+#### ([#592](https://github.com/allegro/hermes/pull/592)) Management: Update Spring Boot (1.4.1) and Jersey (2.23)
+
+#### ([#595](https://github.com/allegro/hermes/pull/595)) Update tech.allegro.schema.json2avro to 0.2.4
+
 ## 0.8.12 (23.09.2016)
 
 ### Features

build.gradle

@@ -42,7 +42,8 @@ allprojects {
     project.ext.versions = [
             guava       : '19.0',
             jackson     : '2.5.1',
-            jersey      : '2.12',
+            jersey      : '2.23.2',
+            jetty       : '9.3.6.v20151106',
             curator     : '2.11.0',
             wiremock    : '1.58',
             fongo       : '1.6.1',

hermes-api/src/main/java/pl/allegro/tech/hermes/api/CrowdGroupDescription.java

@@ -0,0 +1,18 @@
+package pl.allegro.tech.hermes.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CrowdGroupDescription {
+
+    private final String name;
+
+    public CrowdGroupDescription(@JsonProperty("name") String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}

hermes-api/src/main/java/pl/allegro/tech/hermes/api/CrowdGroups.java

@@ -0,0 +1,20 @@
+package pl.allegro.tech.hermes.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CrowdGroups {
+
+    private final List<CrowdGroupDescription> crowdGroupDescriptions;
+
+    public CrowdGroups(@JsonProperty("groups") List<CrowdGroupDescription> crowdGroupDescriptions) {
+        this.crowdGroupDescriptions = crowdGroupDescriptions;
+    }
+
+    public List<CrowdGroupDescription> getCrowdGroupDescriptions() {
+        return crowdGroupDescriptions;
+    }
+}

hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorCode.java

@@ -36,7 +36,8 @@ public enum ErrorCode {
     AVRO_SCHEMA_REMOVAL_DISABLED(BAD_REQUEST),
     SUBSCRIPTION_ENDPOINT_ADDRESS_CHANGE_EXCEPTION(INTERNAL_SERVER_ERROR),
     OAUTH_PROVIDER_NOT_EXISTS(NOT_FOUND),
-    OAUTH_PROVIDER_ALREADY_EXISTS(BAD_REQUEST);
+    OAUTH_PROVIDER_ALREADY_EXISTS(BAD_REQUEST),
+    SUPPORT_TEAMS_COULD_NOT_BE_LOADED(INTERNAL_SERVER_ERROR);
 
     private final int httpCode;
 

hermes-api/src/main/java/pl/allegro/tech/hermes/api/SchemaSource.java → hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchema.java

@@ -5,16 +5,16 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.emptyToNull;
 
-public final class SchemaSource {
+public final class RawSchema {
 
     private final String value;
 
-    private SchemaSource(String value) {
+    private RawSchema(String value) {
         this.value = checkNotNull(emptyToNull(value));
     }
 
-    public static SchemaSource valueOf(String schemaSource) {
-        return new SchemaSource(schemaSource);
+    public static RawSchema valueOf(String schema) {
+        return new RawSchema(schema);
     }
 
     public String value() {
@@ -30,7 +30,7 @@ public boolean equals(Object o) {
             return false;
         }
 
-        SchemaSource that = (SchemaSource) o;
+        RawSchema that = (RawSchema) o;
 
         return value.equals(that.value);
     }
@@ -42,6 +42,6 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return "SchemaSource(" + value + ")";
+        return "RawSource(" + value + ")";
     }
 }

hermes-api/src/main/java/pl/allegro/tech/hermes/api/SupportTeam.java

@@ -0,0 +1,18 @@
+package pl.allegro.tech.hermes.api;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class SupportTeam {
+
+    private final String name;
+
+    @JsonCreator
+    public SupportTeam(@JsonProperty("name") String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}

hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java

@@ -17,10 +17,6 @@ public class Topic {
     @NotNull
     private String description;
 
-    private boolean validationEnabled = false;
-
-    private boolean validationDryRunEnabled = false;
-
     private boolean jsonToAvroDryRunEnabled = false;
 
     @NotNull
@@ -44,14 +40,11 @@ public enum Ack {
     private boolean schemaVersionAwareSerializationEnabled = false;
 
     public Topic(TopicName name, String description, RetentionTime retentionTime,
-                 boolean validationEnabled, boolean validationDryRunEnabled, boolean migratedFromJsonType,
-                 Ack ack, boolean trackingEnabled, ContentType contentType, boolean jsonToAvroDryRunEnabled,
-                 boolean schemaVersionAwareSerializationEnabled) {
+                 boolean migratedFromJsonType, Ack ack, boolean trackingEnabled, ContentType contentType,
+                 boolean jsonToAvroDryRunEnabled, boolean schemaVersionAwareSerializationEnabled) {
         this.name = name;
         this.description = description;
         this.retentionTime = retentionTime;
-        this.validationEnabled = validationEnabled;
-        this.validationDryRunEnabled = validationDryRunEnabled;
         this.ack = (ack == null ? Ack.LEADER : ack);
         this.trackingEnabled = trackingEnabled;
         this.migratedFromJsonType = migratedFromJsonType;
@@ -65,17 +58,14 @@ public Topic(
             @JsonProperty("name") String qualifiedName,
             @JsonProperty("description") String description,
             @JsonProperty("retentionTime") RetentionTime retentionTime,
-            @JsonProperty("validation") boolean validationEnabled,
-            @JsonProperty("validationDryRun") boolean validationDryRunEnabled,
             @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled,
             @JsonProperty("ack") Ack ack,
             @JsonProperty("trackingEnabled") boolean trackingEnabled,
             @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType,
             @JsonProperty("schemaVersionAwareSerializationEnabled") boolean schemaVersionAwareSerializationEnabled,
             @JsonProperty("contentType") ContentType contentType) {
-        this(TopicName.fromQualifiedName(qualifiedName), description, retentionTime, validationEnabled,
-                validationDryRunEnabled, migratedFromJsonType, ack, trackingEnabled, contentType, jsonToAvroDryRunEnabled,
-                schemaVersionAwareSerializationEnabled);
+        this(TopicName.fromQualifiedName(qualifiedName), description, retentionTime, migratedFromJsonType, ack,
+                trackingEnabled, contentType, jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled);
     }
 
     public RetentionTime getRetentionTime() {
@@ -84,8 +74,8 @@ public RetentionTime getRetentionTime() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, description, retentionTime, validationEnabled, validationDryRunEnabled,
-                migratedFromJsonType, trackingEnabled, ack, contentType, jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled);
+        return Objects.hash(name, description, retentionTime, migratedFromJsonType, trackingEnabled, ack, contentType,
+                jsonToAvroDryRunEnabled, schemaVersionAwareSerializationEnabled);
     }
 
     @Override
@@ -101,8 +91,6 @@ public boolean equals(Object obj) {
         return Objects.equals(this.name, other.name)
                 && Objects.equals(this.description, other.description)
                 && Objects.equals(this.retentionTime, other.retentionTime)
-                && Objects.equals(this.isValidationEnabled(), other.isValidationEnabled())
-                && Objects.equals(this.validationDryRunEnabled, other.validationDryRunEnabled)
                 && Objects.equals(this.jsonToAvroDryRunEnabled, other.jsonToAvroDryRunEnabled)
                 && Objects.equals(this.trackingEnabled, other.trackingEnabled)
                 && Objects.equals(this.migratedFromJsonType, other.migratedFromJsonType)
@@ -133,16 +121,6 @@ public void setRetentionTime(RetentionTime retentionTime) {
         this.retentionTime = retentionTime;
     }
 
-    @JsonProperty("validation")
-    public boolean isValidationEnabled() {
-        return validationEnabled || ContentType.AVRO == contentType;
-    }
-
-    @JsonProperty("validationDryRun")
-    public boolean isValidationDryRunEnabled() {
-        return validationDryRunEnabled;
-    }
-
     @JsonProperty("jsonToAvroDryRun")
     public boolean isJsonToAvroDryRunEnabled() {
         return jsonToAvroDryRunEnabled;

hermes-api/src/main/java/pl/allegro/tech/hermes/api/endpoints/SupportTeamsEndpoint.java

@@ -0,0 +1,26 @@
+package pl.allegro.tech.hermes.api.endpoints;
+
+import pl.allegro.tech.hermes.api.SupportTeam;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+import java.util.List;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
+@Path("supportTeams")
+public interface SupportTeamsEndpoint {
+
+    @GET
+    @Produces(APPLICATION_JSON)
+    @Path("/{searchString}")
+    List<SupportTeam> get(@PathParam("searchString") String searchString);
+
+    @GET
+    @Produces(APPLICATION_JSON)
+    @Path("/{searchString}")
+    Response getAsResponse(@PathParam("searchString") String searchString);
+}

hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java

@@ -21,8 +21,6 @@ public void shouldDeserializeTopic() throws Exception {
         assertThat(topic.getName().getName()).isEqualTo("bar");
         assertThat(topic.getName().getGroupName()).isEqualTo("foo");
         assertThat(topic.getDescription()).isEqualTo("description");
-        assertThat(topic.isValidationEnabled()).isFalse();
-        assertThat(topic.isValidationDryRunEnabled()).isFalse();
     }
 
     @Test

hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java

@@ -73,13 +73,13 @@ public void shouldPatchNestedObjects() {
     public void shouldNotResetPrimitiveFields() {
         // given
         Topic topic = topic("group.topic").withTrackingEnabled(true).build();
-        PatchData patch = patchData().set("validation", true).build();
+        PatchData patch = patchData().set("schemaVersionAwareSerializationEnabled", true).build();
 
         // when
         Topic patched = Patch.apply(topic, patch);
 
         // then
         assertThat(patched.isTrackingEnabled()).isTrue();
-        assertThat(patched.isValidationEnabled()).isTrue();
+        assertThat(patched.isSchemaVersionAwareSerializationEnabled()).isTrue();
     }
 }

hermes-common/build.gradle

@@ -3,6 +3,7 @@ apply plugin: 'groovy'
 dependencies {
     compile project(':hermes-api')
     compile project(':hermes-metrics')
+    compile project(':hermes-schema')
 
     compile group: 'com.netflix.archaius', name: 'archaius-core', version: '0.6.0'
 
@@ -14,23 +15,19 @@ dependencies {
         exclude module: 'slf4j-log4j12'
         exclude module: 'log4j'
     }
-    compile (group: 'com.github.fge', name: 'json-schema-validator', version: '2.2.6') {
-        exclude group: 'net.sf.jopt-simple'
-    }
 
     compile group: 'org.glassfish.hk2', name: 'hk2-locator', version: '2.3.0-b01'
     compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: versions.jersey
     compile group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: versions.jersey
     compile group: 'org.glassfish.jersey.ext', name: 'jersey-bean-validation', version: versions.jersey
 
-    compile group: 'tech.allegro.schema.json2avro', name: 'converter', version: '0.2.2'
+    compile group: 'tech.allegro.schema.json2avro', name: 'converter', version: '0.2.4'
 
     compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.0'
     compile group: 'commons-codec', name: 'commons-codec', version: '1.9'
     compile group: 'com.google.guava', name: 'guava', version: versions.guava
 
     compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: versions.jackson
-    compile group: 'io.fastjson', name: 'boon', version: '0.28'
     compile group: 'org.apache.avro', name: 'avro', version: '1.7.7'
 
     compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '3.1.1'
@@ -41,13 +38,8 @@ dependencies {
     compile group: 'org.slf4j', name: 'log4j-over-slf4j', version: '1.7.10'
     compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.1.3'
 
-    compile(group: 'org.apache.kafka', name: 'kafka_2.11', version: '0.8.2.1') {
-        exclude group: 'javax.jms'
-        exclude group: 'com.sun.jdmk'
-        exclude group: 'com.sun.jmx'
+    compile(group: 'org.apache.kafka', name: 'kafka_2.11', version: '0.10.0.1') {
         exclude group: 'net.sf.jopt-simple'
-        exclude module: 'slf4j-log4j12'
-        exclude module: 'log4j'
     }
 
     testCompile project(':hermes-test-helper')