Skip to content

Commit

Permalink
Use dockerized Redis in tests
Browse files Browse the repository at this point in the history 
Additionally, move redis dependencies to presto-redis from root pom.xml.
  • Loading branch information
@ebyhr
ebyhr committed Jan 27, 2020
1 parent 0bc2584 commit bd5df5d
Show file tree
Hide file tree
Showing 9 changed files with 61 additions and 94 deletions.
12 changes: 0 additions & 12 deletions pom.xml
View file
Expand Up @@ -1050,18 +1050,6 @@
<version>0.9.0</version>
</dependency>

<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.6.2</version>
</dependency>

<dependency>
<groupId>com.orange.redis-embedded</groupId>
<artifactId>embedded-redis</artifactId>
<version>0.6</version>
</dependency>

<dependency>
<groupId>io.prestosql.tempto</groupId>
<artifactId>tempto-core</artifactId>
Expand Down
5 changes: 3 additions & 2 deletions presto-redis/pom.xml
View file
Expand Up @@ -71,6 +71,7 @@
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.6.2</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -167,8 +168,8 @@
</dependency>

<dependency>
<groupId>com.orange.redis-embedded</groupId>
<artifactId>embedded-redis</artifactId>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>

Expand Down
22 changes: 10 additions & 12 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/RedisQueryRunner.java
View file
Expand Up @@ -21,7 +21,7 @@
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.plugin.redis.util.CodecSupplier;
import io.prestosql.plugin.redis.util.EmbeddedRedis;
import io.prestosql.plugin.redis.util.RedisServer;
import io.prestosql.plugin.redis.util.RedisTestUtils;
import io.prestosql.plugin.tpch.TpchPlugin;
import io.prestosql.spi.connector.SchemaTableName;
Expand All @@ -47,13 +47,13 @@ private RedisQueryRunner() {}
private static final Logger log = Logger.get(RedisQueryRunner.class);
private static final String TPCH_SCHEMA = "tpch";

public static DistributedQueryRunner createRedisQueryRunner(EmbeddedRedis embeddedRedis, String dataFormat, TpchTable<?>... tables)
public static DistributedQueryRunner createRedisQueryRunner(RedisServer redisServer, String dataFormat, TpchTable<?>... tables)
throws Exception
{
return createRedisQueryRunner(embeddedRedis, dataFormat, ImmutableList.copyOf(tables));
return createRedisQueryRunner(redisServer, dataFormat, ImmutableList.copyOf(tables));
}

public static DistributedQueryRunner createRedisQueryRunner(EmbeddedRedis embeddedRedis, String dataFormat, Iterable<TpchTable<?>> tables)
public static DistributedQueryRunner createRedisQueryRunner(RedisServer redisServer, String dataFormat, Iterable<TpchTable<?>> tables)
throws Exception
{
DistributedQueryRunner queryRunner = null;
Expand All @@ -63,35 +63,33 @@ public static DistributedQueryRunner createRedisQueryRunner(EmbeddedRedis embedd
queryRunner.installPlugin(new TpchPlugin());
queryRunner.createCatalog("tpch", "tpch");

embeddedRedis.start();

Map<SchemaTableName, RedisTableDescription> tableDescriptions = createTpchTableDescriptions(queryRunner.getCoordinator().getMetadata(), tables, dataFormat);

installRedisPlugin(embeddedRedis, queryRunner, tableDescriptions);
installRedisPlugin(redisServer, queryRunner, tableDescriptions);

TestingPrestoClient prestoClient = queryRunner.getClient();

log.info("Loading data...");
long startTime = System.nanoTime();
for (TpchTable<?> table : tables) {
loadTpchTable(embeddedRedis, prestoClient, table, dataFormat);
loadTpchTable(redisServer, prestoClient, table, dataFormat);
}
log.info("Loading complete in %s", nanosSince(startTime).toString(SECONDS));
embeddedRedis.destroyJedisPool();
redisServer.destroyJedisPool();
return queryRunner;
}
catch (Throwable e) {
closeAllSuppress(e, queryRunner, embeddedRedis);
closeAllSuppress(e, queryRunner, redisServer);
throw e;
}
}

private static void loadTpchTable(EmbeddedRedis embeddedRedis, TestingPrestoClient prestoClient, TpchTable<?> table, String dataFormat)
private static void loadTpchTable(RedisServer redisServer, TestingPrestoClient prestoClient, TpchTable<?> table, String dataFormat)
{
long start = System.nanoTime();
log.info("Running import for %s", table.getTableName());
RedisTestUtils.loadTpchTable(
embeddedRedis,
redisServer,
prestoClient,
redisTableName(table),
new QualifiedObjectName("tpch", TINY_SCHEMA_NAME, table.getTableName().toLowerCase(ENGLISH)),
Expand Down
16 changes: 8 additions & 8 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/TestMinimalFunctionality.java
View file
Expand Up @@ -17,8 +17,8 @@
import io.prestosql.Session;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.metadata.TableHandle;
import io.prestosql.plugin.redis.util.EmbeddedRedis;
import io.prestosql.plugin.redis.util.JsonEncoder;
import io.prestosql.plugin.redis.util.RedisServer;
import io.prestosql.security.AllowAllAccessControl;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.type.BigintType;
Expand Down Expand Up @@ -49,23 +49,23 @@ public class TestMinimalFunctionality
.setSchema("default")
.build();

private EmbeddedRedis embeddedRedis;
private RedisServer redisServer;
private String tableName;
private StandaloneQueryRunner queryRunner;

@BeforeClass
public void startRedis()
throws Exception
{
embeddedRedis = EmbeddedRedis.createEmbeddedRedis();
embeddedRedis.start();
redisServer = new RedisServer();
}

@AfterClass(alwaysRun = true)
public void stopRedis()
throws Exception
{
embeddedRedis.close();
embeddedRedis = null;
redisServer.close();
redisServer = null;
}

@BeforeMethod
Expand All @@ -76,7 +76,7 @@ public void spinUp()

this.queryRunner = new StandaloneQueryRunner(SESSION);

installRedisPlugin(embeddedRedis, queryRunner,
installRedisPlugin(redisServer, queryRunner,
ImmutableMap.<SchemaTableName, RedisTableDescription>builder()
.put(createEmptyTableDescription(new SchemaTableName("default", tableName)))
.build());
Expand All @@ -94,7 +94,7 @@ private void populateData(int count)
JsonEncoder jsonEncoder = new JsonEncoder();
for (long i = 0; i < count; i++) {
Object value = ImmutableMap.of("id", Long.toString(i), "value", UUID.randomUUID().toString());
try (Jedis jedis = embeddedRedis.getJedisPool().getResource()) {
try (Jedis jedis = redisServer.getJedisPool().getResource()) {
jedis.set(tableName + ":" + i, jsonEncoder.toString(value));
}
}
Expand Down
13 changes: 6 additions & 7 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/TestRedisDistributed.java
View file
Expand Up @@ -13,32 +13,31 @@
*/
package io.prestosql.plugin.redis;

import io.prestosql.plugin.redis.util.EmbeddedRedis;
import io.prestosql.plugin.redis.util.RedisServer;
import io.prestosql.testing.AbstractTestQueries;
import io.prestosql.testing.QueryRunner;
import io.prestosql.tpch.TpchTable;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;

import static io.prestosql.plugin.redis.util.EmbeddedRedis.createEmbeddedRedis;

@Test
public class TestRedisDistributed
extends AbstractTestQueries
{
private EmbeddedRedis embeddedRedis;
private RedisServer redisServer;

@Override
protected QueryRunner createQueryRunner()
throws Exception
{
embeddedRedis = createEmbeddedRedis();
return RedisQueryRunner.createRedisQueryRunner(embeddedRedis, "string", TpchTable.getTables());
redisServer = new RedisServer();
return RedisQueryRunner.createRedisQueryRunner(redisServer, "string", TpchTable.getTables());
}

@AfterClass(alwaysRun = true)
public void destroy()
throws Exception
{
embeddedRedis.close();
redisServer.close();
}
}
12 changes: 6 additions & 6 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/TestRedisDistributedHash.java
View file
Expand Up @@ -13,33 +13,33 @@
*/
package io.prestosql.plugin.redis;

import io.prestosql.plugin.redis.util.EmbeddedRedis;
import io.prestosql.plugin.redis.util.RedisServer;
import io.prestosql.testing.AbstractTestQueries;
import io.prestosql.testing.QueryRunner;
import io.prestosql.tpch.TpchTable;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;

import static io.prestosql.plugin.redis.RedisQueryRunner.createRedisQueryRunner;
import static io.prestosql.plugin.redis.util.EmbeddedRedis.createEmbeddedRedis;

@Test
public class TestRedisDistributedHash
extends AbstractTestQueries
{
private EmbeddedRedis embeddedRedis;
private RedisServer redisServer;

@Override
protected QueryRunner createQueryRunner()
throws Exception
{
embeddedRedis = createEmbeddedRedis();
return createRedisQueryRunner(embeddedRedis, "hash", TpchTable.getTables());
redisServer = new RedisServer();
return createRedisQueryRunner(redisServer, "hash", TpchTable.getTables());
}

@AfterClass(alwaysRun = true)
public void destroy()
throws Exception
{
embeddedRedis.close();
redisServer.close();
}
}
12 changes: 6 additions & 6 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/TestRedisIntegrationSmokeTest.java
View file
Expand Up @@ -13,7 +13,7 @@
*/
package io.prestosql.plugin.redis;

import io.prestosql.plugin.redis.util.EmbeddedRedis;
import io.prestosql.plugin.redis.util.RedisServer;
import io.prestosql.testing.AbstractTestIntegrationSmokeTest;
import io.prestosql.testing.QueryRunner;
import io.prestosql.testing.sql.TestTable;
Expand All @@ -22,27 +22,27 @@
import org.testng.annotations.Test;

import static io.prestosql.plugin.redis.RedisQueryRunner.createRedisQueryRunner;
import static io.prestosql.plugin.redis.util.EmbeddedRedis.createEmbeddedRedis;
import static io.prestosql.tpch.TpchTable.ORDERS;

@Test
public class TestRedisIntegrationSmokeTest
extends AbstractTestIntegrationSmokeTest
{
private EmbeddedRedis embeddedRedis;
private RedisServer redisServer;

@Override
protected QueryRunner createQueryRunner()
throws Exception
{
embeddedRedis = createEmbeddedRedis();
return createRedisQueryRunner(embeddedRedis, "string", ORDERS);
redisServer = new RedisServer();
return createRedisQueryRunner(redisServer, "string", ORDERS);
}

@AfterClass(alwaysRun = true)
public void destroy()
throws Exception
{
embeddedRedis.close();
redisServer.close();
}

@Override
Expand Down
55 changes: 18 additions & 37 deletions ...osql/plugin/redis/util/EmbeddedRedis.java → ...stosql/plugin/redis/util/RedisServer.java
View file
Expand Up @@ -13,37 +13,28 @@
*/
package io.prestosql.plugin.redis.util;

import com.google.common.net.HostAndPort;
import org.testcontainers.containers.GenericContainer;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.embedded.RedisServer;

import java.io.Closeable;
import java.io.IOException;
import java.net.URISyntaxException;

public class EmbeddedRedis
public class RedisServer
implements Closeable
{
private final RedisServer redisServer;
private JedisPool jedisPool;
private static final int PORT = 6379;

public static EmbeddedRedis createEmbeddedRedis()
throws IOException, URISyntaxException
{
return new EmbeddedRedis();
}
private final GenericContainer<?> container;
private final JedisPool jedisPool;

EmbeddedRedis()
throws IOException, URISyntaxException
public RedisServer()
{
redisServer = new RedisServer();
}
container = new GenericContainer<>("redis:2.8.9")
.withExposedPorts(PORT);
container.start();

public void start()
throws IOException
{
redisServer.start();
jedisPool = new JedisPool(new JedisPoolConfig(), getConnectString(), getPort());
jedisPool = new JedisPool(container.getContainerIpAddress(), container.getMappedPort(PORT));
}

public JedisPool getJedisPool()
Expand All @@ -56,26 +47,16 @@ public void destroyJedisPool()
jedisPool.destroy();
}

@Override
public void close()
public HostAndPort getHostAndPort()
{
jedisPool.destroy();
try {
redisServer.stop();
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
return HostAndPort.fromParts(container.getContainerIpAddress(), container.getMappedPort(PORT));
}

public int getPort()
{
return redisServer.getPort();
}

public String getConnectString()
@Override
public void close()
throws IOException
{
return "localhost";
jedisPool.destroy();
container.close();
}
}
8 changes: 4 additions & 4 deletions presto-redis/src/test/java/io/prestosql/plugin/redis/util/RedisTestUtils.java
View file
Expand Up @@ -35,24 +35,24 @@ public final class RedisTestUtils
{
private RedisTestUtils() {}

public static void installRedisPlugin(EmbeddedRedis embeddedRedis, QueryRunner queryRunner, Map<SchemaTableName, RedisTableDescription> tableDescriptions)
public static void installRedisPlugin(RedisServer redisServer, QueryRunner queryRunner, Map<SchemaTableName, RedisTableDescription> tableDescriptions)
{
RedisPlugin redisPlugin = new RedisPlugin();
redisPlugin.setTableDescriptionSupplier(() -> tableDescriptions);
queryRunner.installPlugin(redisPlugin);

Map<String, String> redisConfig = ImmutableMap.of(
"redis.nodes", embeddedRedis.getConnectString() + ":" + embeddedRedis.getPort(),
"redis.nodes", redisServer.getHostAndPort().toString(),
"redis.table-names", Joiner.on(",").join(tableDescriptions.keySet()),
"redis.default-schema", "default",
"redis.hide-internal-columns", "true",
"redis.key-prefix-schema-table", "true");
queryRunner.createCatalog("redis", "redis", redisConfig);
}

public static void loadTpchTable(EmbeddedRedis embeddedRedis, TestingPrestoClient prestoClient, String tableName, QualifiedObjectName tpchTableName, String dataFormat)
public static void loadTpchTable(RedisServer redisServer, TestingPrestoClient prestoClient, String tableName, QualifiedObjectName tpchTableName, String dataFormat)
{
RedisLoader tpchLoader = new RedisLoader(prestoClient.getServer(), prestoClient.getDefaultSession(), embeddedRedis.getJedisPool(), tableName, dataFormat);
RedisLoader tpchLoader = new RedisLoader(prestoClient.getServer(), prestoClient.getDefaultSession(), redisServer.getJedisPool(), tableName, dataFormat);
tpchLoader.execute(format("SELECT * from %s", tpchTableName));
}

Expand Down

0 comments on commit bd5df5d

Please sign in to comment.