From 8a1334c8a4552008fba4a9e819010fb498ade296 Mon Sep 17 00:00:00 2001 From: Shriram Kumar Date: Wed, 20 Sep 2017 12:24:28 -0700 Subject: [PATCH 1/2] Helper methods and a RandomPool utility class. --- pom.xml | 2 +- .../java/com/yahoo/bullet/RandomPool.java | 68 +++++++++++++++ .../com/yahoo/bullet/pubsub/Metadata.java | 5 ++ .../java/com/yahoo/bullet/pubsub/PubSub.java | 49 +++++++++-- .../yahoo/bullet/pubsub/PubSubException.java | 17 ++++ .../yahoo/bullet/pubsub/PubSubMessage.java | 7 ++ .../com/yahoo/bullet/pubsub/Publisher.java | 5 ++ .../com/yahoo/bullet/pubsub/Subscriber.java | 5 ++ src/main/resources/checkstyle.xml | 7 ++ src/main/resources/copyright.txt | 5 ++ .../java/com/yahoo/bullet/RandomPoolTest.java | 83 +++++++++++++++++++ .../com/yahoo/bullet/pubsub/MetadataTest.java | 5 ++ .../com/yahoo/bullet/pubsub/MockPubSub.java | 7 +- .../bullet/pubsub/PubSubExceptionTest.java | 5 ++ .../bullet/pubsub/PubSubMessageTest.java | 5 ++ .../com/yahoo/bullet/pubsub/PubSubTest.java | 5 ++ .../yahoo/bullet/pubsub/PublisherTest.java | 5 ++ .../yahoo/bullet/pubsub/SubscriberTest.java | 5 ++ 18 files changed, 280 insertions(+), 10 deletions(-) create mode 100644 src/main/java/com/yahoo/bullet/RandomPool.java create mode 100644 src/main/resources/copyright.txt create mode 100644 src/test/java/com/yahoo/bullet/RandomPoolTest.java diff --git a/pom.xml b/pom.xml index e097c5f1..f21631ec 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.yahoo.bullet bullet-core - 0.2.2-SNAPSHOT + 0.2.3-SNAPSHOT jar bullet-core diff --git a/src/main/java/com/yahoo/bullet/RandomPool.java b/src/main/java/com/yahoo/bullet/RandomPool.java new file mode 100644 index 00000000..6090e372 --- /dev/null +++ b/src/main/java/com/yahoo/bullet/RandomPool.java @@ -0,0 +1,68 @@ +/* + * Copyright 2016, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ +package com.yahoo.bullet; + +import java.util.List; +import java.util.Random; + +public class RandomPool { + private List items; + + private static final Random RANDOM = new Random(); + + /** + * Constructor for the RandomPool that takes a list of items. + * @param items A list of items to form the pool with. + */ + public RandomPool(List items) { + this.items = items; + } + + /** + * Get a random item from the pool. + * + * @return a randomly chosen item from the pool. + */ + public T get() { + if (items == null || items.isEmpty()) { + return null; + } + return items.get(RANDOM.nextInt(items.size())); + } + + /** + * Clear the RandomPool. Gets now return null. + */ + public void clear() { + items = null; + } + + @Override + public String toString() { + return items == null ? null : items.toString(); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null) { + return false; + } + if (!(object instanceof RandomPool)) { + return false; + } + RandomPool asPool = (RandomPool) object; + return items == null ? asPool.items == null : items.equals(asPool.items); + } + + @Override + public int hashCode() { + // Any number would do since we want RandomPools of null to be equal to each other. + return items == null ? 42 : items.hashCode(); + } +} diff --git a/src/main/java/com/yahoo/bullet/pubsub/Metadata.java b/src/main/java/com/yahoo/bullet/pubsub/Metadata.java index e6342f15..e430f482 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/Metadata.java +++ b/src/main/java/com/yahoo/bullet/pubsub/Metadata.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import lombok.AllArgsConstructor; diff --git a/src/main/java/com/yahoo/bullet/pubsub/PubSub.java b/src/main/java/com/yahoo/bullet/pubsub/PubSub.java index 8d42e463..f9aa27c4 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/PubSub.java +++ b/src/main/java/com/yahoo/bullet/pubsub/PubSub.java @@ -1,10 +1,15 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import com.yahoo.bullet.BulletConfig; -import java.io.Serializable; import java.lang.reflect.Constructor; import java.util.List; +import java.util.Objects; /** * Notation: Partition is a unit of parallelism in the Pub/Sub queue. @@ -12,7 +17,7 @@ * Implementations of PubSub should take in a {@link BulletConfig} and use the information to wire up and return * Publishers and Subscribers. */ -public abstract class PubSub implements Serializable { +public abstract class PubSub { /** * The context determines how the {@link Publisher} and {@link Subscriber} returned by PubSub behave. For example, * If the Context is {@link Context#QUERY_SUBMISSION}: @@ -28,14 +33,21 @@ public enum Context { } protected Context context; + protected BulletConfig config; /** * Instantiate a PubSub using parameters from {@link BulletConfig}. * * @param config The {@link BulletConfig} containing all required PubSub parameters. + * @throws PubSubException if the context name is not present or cannot be parsed. */ - public PubSub(BulletConfig config) { - context = Context.valueOf(config.get(BulletConfig.PUBSUB_CONTEXT_NAME).toString()); + public PubSub(BulletConfig config) throws PubSubException { + this.config = config; + try { + this.context = Context.valueOf(getRequiredConfig(String.class, BulletConfig.PUBSUB_CONTEXT_NAME)); + } catch (RuntimeException e) { + throw new PubSubException("Cannot create PubSub", e); + } } /** @@ -43,8 +55,9 @@ public PubSub(BulletConfig config) { * {@link PubSub#context}). * * @return {@link Publisher} wired as required. + * @throws PubSubException if the Publisher could not be created. */ - public abstract Publisher getPublisher(); + public abstract Publisher getPublisher() throws PubSubException; /** * Get a list of n {@link Publisher} instances with the allocated partitions in the appropriate queue @@ -52,16 +65,18 @@ public PubSub(BulletConfig config) { * * @param n The number of Publishers requested. * @return The {@link List} of n Publishers wired as required. + * @throws PubSubException if Publishers could not be created. */ - public abstract List getPublishers(int n); + public abstract List getPublishers(int n) throws PubSubException; /** * Get a {@link Subscriber} instance wired to read from all allocated partitions in the appropriate queue (See * {@link PubSub#context}). * * @return {@link Subscriber} wired as required. + * @throws PubSubException if the Subscriber could not be created. */ - public abstract Subscriber getSubscriber(); + public abstract Subscriber getSubscriber() throws PubSubException; /** * Get a list of n {@link Subscriber} instances with allocated partitions from the appropriate queue @@ -69,8 +84,9 @@ public PubSub(BulletConfig config) { * * @param n The number of Subscribers requested. * @return The {@link List} of n Subscribers wired as required. + * @throws PubSubException if Subscribers could not be created. */ - public abstract List getSubscribers(int n); + public abstract List getSubscribers(int n) throws PubSubException; /** * Create a PubSub instance using the class specified in the config file. @@ -89,4 +105,21 @@ public static PubSub from(BulletConfig config) throws PubSubException { throw new PubSubException("Cannot create PubSub instance.", e); } } + + /** + * A method to get a required configuration of a particular type. + * + * @param name The name of the required configuration. + * @param tClass The class of the required configuration. + * @param The type to cast the configuration to. Inferred from tClass. + * @return The extracted configuration of type T. + * @throws PubSubException if the configuration is missing or cannot be cast to type T. + */ + public T getRequiredConfig(Class tClass, String name) throws PubSubException { + try { + return (T) Objects.requireNonNull(config.get(name)); + } catch (Exception e) { + throw PubSubException.forArgument(name, e); + } + } } diff --git a/src/main/java/com/yahoo/bullet/pubsub/PubSubException.java b/src/main/java/com/yahoo/bullet/pubsub/PubSubException.java index 240bb131..69a9c0bd 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/PubSubException.java +++ b/src/main/java/com/yahoo/bullet/pubsub/PubSubException.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; /** @@ -22,4 +27,16 @@ public PubSubException(String message) { public PubSubException(String message, Throwable cause) { super(message, cause); } + + /** + * Method to create a PubSubException when a required argument could not be read. + * + * @param name The name of the argument that could not be read. + * @param cause The optional {@link Throwable} that caused the exception. + * @return A PubSubException indicating failure to read a required argument. + */ + public static PubSubException forArgument(String name, Throwable cause) { + String message = "Could not read required argument: " + name; + return cause == null ? new PubSubException(message) : new PubSubException(message, cause); + } } diff --git a/src/main/java/com/yahoo/bullet/pubsub/PubSubMessage.java b/src/main/java/com/yahoo/bullet/pubsub/PubSubMessage.java index 44e9c475..adebcbcc 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/PubSubMessage.java +++ b/src/main/java/com/yahoo/bullet/pubsub/PubSubMessage.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import com.yahoo.bullet.pubsub.Metadata.Signal; @@ -13,6 +18,8 @@ */ @Getter public class PubSubMessage implements Serializable { + private static final long serialVersionUID = 2407848310969237888L; + private String id; private int sequence; private String content; diff --git a/src/main/java/com/yahoo/bullet/pubsub/Publisher.java b/src/main/java/com/yahoo/bullet/pubsub/Publisher.java index 916668ab..fdf2ba37 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/Publisher.java +++ b/src/main/java/com/yahoo/bullet/pubsub/Publisher.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; public interface Publisher { diff --git a/src/main/java/com/yahoo/bullet/pubsub/Subscriber.java b/src/main/java/com/yahoo/bullet/pubsub/Subscriber.java index bba801eb..79385165 100644 --- a/src/main/java/com/yahoo/bullet/pubsub/Subscriber.java +++ b/src/main/java/com/yahoo/bullet/pubsub/Subscriber.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; public interface Subscriber { diff --git a/src/main/resources/checkstyle.xml b/src/main/resources/checkstyle.xml index cd18cbc0..3d306c7a 100644 --- a/src/main/resources/checkstyle.xml +++ b/src/main/resources/checkstyle.xml @@ -14,6 +14,13 @@ + + + + + + + diff --git a/src/main/resources/copyright.txt b/src/main/resources/copyright.txt new file mode 100644 index 00000000..b18622f5 --- /dev/null +++ b/src/main/resources/copyright.txt @@ -0,0 +1,5 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ diff --git a/src/test/java/com/yahoo/bullet/RandomPoolTest.java b/src/test/java/com/yahoo/bullet/RandomPoolTest.java new file mode 100644 index 00000000..5b938050 --- /dev/null +++ b/src/test/java/com/yahoo/bullet/RandomPoolTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2016, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ +package com.yahoo.bullet; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class RandomPoolTest { + @Test + public void testDefaultOverrides() { + RandomPool poolA = new RandomPool<>(null); + RandomPool poolB = new RandomPool<>(null); + Assert.assertTrue(poolA.equals(poolA)); + Assert.assertEquals(poolA.hashCode(), poolA.hashCode()); + + Assert.assertFalse(poolA.equals(null)); + Assert.assertFalse(poolA.equals("foo")); + + Assert.assertTrue(poolA.equals(poolB)); + Assert.assertEquals(poolA.hashCode(), poolB.hashCode()); + + poolA = new RandomPool<>(Collections.singletonList("foo")); + poolB = new RandomPool<>(Collections.singletonList("foo")); + Assert.assertTrue(poolA.equals(poolB)); + Assert.assertEquals(poolA.hashCode(), poolB.hashCode()); + + poolB = new RandomPool<>(Collections.singletonList("bar")); + Assert.assertFalse(poolA.equals(poolB)); + + List contents = Collections.singletonList("foo"); + poolA = new RandomPool<>(contents); + poolB = new RandomPool<>(contents); + Assert.assertTrue(poolA.equals(poolB)); + Assert.assertEquals(poolA.hashCode(), poolB.hashCode()); + } + + @Test + public void testToString() { + RandomPool pool = new RandomPool<>(null); + Assert.assertNull(pool.toString()); + pool = new RandomPool<>(Collections.singletonList("foo")); + Assert.assertEquals(pool.toString(), Collections.singletonList("foo").toString()); + } + + @Test + public void testEmptyCase() { + RandomPool pool = new RandomPool<>(null); + Assert.assertNull(pool.get()); + pool = new RandomPool<>(Collections.emptyList()); + Assert.assertNull(pool.get()); + } + + @Test + public void testRandomGet() { + List list = Arrays.asList(1, 3, 4); + Map map = list.stream().collect(Collectors.toMap(Function.identity(), x -> 0)); + RandomPool pool = new RandomPool<>(list); + for (int i = 0; i < 1000; ++i) { + int item = pool.get(); + map.put(item, map.get(item) + 1); + } + // That this is false is 1 - (2/3)^1000 + Assert.assertTrue(map.values().stream().allMatch(v -> v > 0)); + } + + @Test + public void testGetReturnsNullAfterClear() { + List list = Arrays.asList(1, 3, 4); + RandomPool pool = new RandomPool<>(list); + pool.clear(); + Assert.assertNull(pool.get()); + } +} diff --git a/src/test/java/com/yahoo/bullet/pubsub/MetadataTest.java b/src/test/java/com/yahoo/bullet/pubsub/MetadataTest.java index 98de0774..5b50789a 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/MetadataTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/MetadataTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import org.testng.Assert; diff --git a/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java b/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java index a22fe805..d08e8a4d 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java +++ b/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import com.yahoo.bullet.BulletConfig; @@ -11,7 +16,7 @@ class MockPubSub extends PubSub { public static final String MOCK_MESSAGE_NAME = "MOCK_MESSAGE"; private String mockMessage; - public MockPubSub(BulletConfig config) { + public MockPubSub(BulletConfig config) throws PubSubException { super(config); mockMessage = config.get(MOCK_MESSAGE_NAME).toString(); } diff --git a/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java b/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java index a2ce4166..f86320b9 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import org.testng.Assert; diff --git a/src/test/java/com/yahoo/bullet/pubsub/PubSubMessageTest.java b/src/test/java/com/yahoo/bullet/pubsub/PubSubMessageTest.java index d965d1ed..a494cc3e 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PubSubMessageTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PubSubMessageTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import com.yahoo.bullet.pubsub.Metadata.Signal; diff --git a/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java b/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java index 2fe02662..7d2b32b6 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import com.yahoo.bullet.BulletConfig; diff --git a/src/test/java/com/yahoo/bullet/pubsub/PublisherTest.java b/src/test/java/com/yahoo/bullet/pubsub/PublisherTest.java index 17ec4a54..b213c02d 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PublisherTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PublisherTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import org.testng.Assert; diff --git a/src/test/java/com/yahoo/bullet/pubsub/SubscriberTest.java b/src/test/java/com/yahoo/bullet/pubsub/SubscriberTest.java index 4fd22a31..5ebf6c54 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/SubscriberTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/SubscriberTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2017, Yahoo Inc. + * Licensed under the terms of the Apache License, Version 2.0. + * See the LICENSE file associated with the project for terms. + */ package com.yahoo.bullet.pubsub; import lombok.NoArgsConstructor; From 1974fb8d4e9d386230691ab04c84346243b0a45e Mon Sep 17 00:00:00 2001 From: Shriram Kumar Date: Wed, 20 Sep 2017 12:51:35 -0700 Subject: [PATCH 2/2] Adding tests --- .../com/yahoo/bullet/pubsub/MockPubSub.java | 2 +- .../bullet/pubsub/PubSubExceptionTest.java | 19 ++++++++++++++++--- .../com/yahoo/bullet/pubsub/PubSubTest.java | 9 +++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java b/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java index d08e8a4d..f457a334 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java +++ b/src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java @@ -18,7 +18,7 @@ class MockPubSub extends PubSub { public MockPubSub(BulletConfig config) throws PubSubException { super(config); - mockMessage = config.get(MOCK_MESSAGE_NAME).toString(); + mockMessage = getRequiredConfig(String.class, MOCK_MESSAGE_NAME); } @Override diff --git a/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java b/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java index f86320b9..6928f1f4 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java @@ -8,13 +8,26 @@ import org.testng.Assert; import org.testng.annotations.Test; -import java.util.UUID; - public class PubSubExceptionTest { @Test public void testGetMessage() { - String randomMessage = UUID.randomUUID().toString(); + String randomMessage = "foo"; PubSubException ex = new PubSubException(randomMessage); Assert.assertTrue(ex.getMessage().equals(randomMessage)); } + + @Test + public void testGetArgumentFailedWithoutCause() { + PubSubException ex = PubSubException.forArgument("bar", null); + Assert.assertEquals(ex.getMessage(), "Could not read required argument: bar"); + Assert.assertNull(ex.getCause()); + } + + @Test + public void testGetArgumentFailedWithCause() { + Throwable cause = new NullPointerException(); + PubSubException ex = PubSubException.forArgument("bar", cause); + Assert.assertEquals(ex.getMessage(), "Could not read required argument: bar"); + Assert.assertEquals(ex.getCause(), cause); + } } diff --git a/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java b/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java index 7d2b32b6..be46ba2d 100644 --- a/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java +++ b/src/test/java/com/yahoo/bullet/pubsub/PubSubTest.java @@ -9,6 +9,7 @@ import org.testng.Assert; import org.testng.annotations.Test; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.UUID; @@ -45,4 +46,12 @@ public void testIllegalPubSubClassName() throws Exception { throw e; } } + + @Test(expectedExceptions = PubSubException.class) + public void testConstructorRuntimeExceptionThrowsPubSubException() throws IOException, PubSubException { + BulletConfig config = new BulletConfig("src/test/resources/test_config.yaml"); + config.set(MockPubSub.MOCK_MESSAGE_NAME, ""); + config.set(BulletConfig.PUBSUB_CONTEXT_NAME, ""); + PubSub.from(config); + } }