Skip to content

Commit

Permalink
SimpleSeedProvider can resolve multiple IP addresses per DNS record
Browse files Browse the repository at this point in the history
patch by Ben Bromhead; reviewed by Stefan Miklosovic and Andrés de la Peña for CASSANDRA-14361

Co-authored-by: Stefan Miklosovic <smiklosovic@apache.org>
  • Loading branch information
bbromhead and smiklosovic committed Jan 18, 2023
1 parent 1bba8eb commit b07c312
Show file tree
Hide file tree
Showing 9 changed files with 286 additions and 6 deletions.
4 changes: 4 additions & 0 deletions .build/cassandra-build-deps-template.xml
Expand Up @@ -38,6 +38,10 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions .build/parent-pom-template.xml
Expand Up @@ -455,6 +455,12 @@
<version>4.7.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>4.7.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.quicktheories</groupId>
<artifactId>quicktheories</artifactId>
Expand Down
1 change: 1 addition & 0 deletions CHANGES.txt
@@ -1,4 +1,5 @@
4.2
* SimpleSeedProvider can resolve multiple IP addresses per DNS record (CASSANDRA-14361)
* Remove mocking in InternalNodeProbe spying on StorageServiceMBean (CASSANDRA-18152)
* Add compaction_properties column to system.compaction_history table and nodetool compactionhistory command (CASSANDRA-18061)
* Remove ProtocolVersion entirely from the CollectionSerializer ecosystem (CASSANDRA-18114)
Expand Down
4 changes: 4 additions & 0 deletions NEWS.txt
Expand Up @@ -111,6 +111,10 @@ New features
aggregation functions, but they operate on the elements of collection columns. The new functions are `map_keys`,
`map_values`, `collection_count`, `collection_min`, `collection_max`, `collection_sum` and `collection_avg`.
- Added compaction_properties column to system.compaction_history table and nodetool compactionhistory command
- SimpleSeedProvider can resolve multiple IP addresses per DNS record. SimpleSeedProvider reacts on
the paramater called `resolve_multiple_ip_addresses_per_dns_record` which value is meant to be boolean and by
default it is set to false. When set to true, SimpleSeedProvider will resolve all IP addresses per DNS record,
based on the configured name service on the system.

Upgrading
---------
Expand Down
3 changes: 3 additions & 0 deletions conf/cassandra.yaml
Expand Up @@ -578,6 +578,9 @@ seed_provider:
# seeds is actually a comma-delimited list of addresses.
# Ex: "<ip1>,<ip2>,<ip3>"
- seeds: "127.0.0.1:7000"
# If set to "true", SimpleSeedProvider will return all IP addresses for a DNS name,
# based on the configured name service on the system. Defaults to "false".
# resolve_multiple_ip_addresses_per_dns_record: "false"

# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
Expand Down
Expand Up @@ -331,8 +331,14 @@ public enum CassandraRelevantProperties
*
* If only keyspaces are specified, mutations for all tables in such keyspace will be replayed
* */
COMMIT_LOG_REPLAY_LIST("cassandra.replayList", null)
COMMIT_LOG_REPLAY_LIST("cassandra.replayList", null),

/**
* The maximum number of seeds returned by a seed provider before emmitting a warning.
* A large seed list may impact effectiveness of the third gossip round.
* The default used in SimpleSeedProvider is 20.
*/
SEED_COUNT_WARN_THRESHOLD("cassandra.seed_count_warn_threshold"),
;

CassandraRelevantProperties(String key, String defaultVal)
Expand Down
32 changes: 31 additions & 1 deletion src/java/org/apache/cassandra/locator/InetAddressAndPort.java
Expand Up @@ -26,7 +26,11 @@
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.regex.Pattern;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;

Expand Down Expand Up @@ -64,7 +68,8 @@ public final class InetAddressAndPort extends InetSocketAddress implements Compa

public final byte[] addressBytes;

private InetAddressAndPort(InetAddress address, byte[] addressBytes, int port)
@VisibleForTesting
InetAddressAndPort(InetAddress address, byte[] addressBytes, int port)
{
super(address, port);
Preconditions.checkNotNull(address);
Expand Down Expand Up @@ -215,6 +220,31 @@ public static InetAddressAndPort getByName(String name) throws UnknownHostExcept
return getByNameOverrideDefaults(name, null);
}


public static List<InetAddressAndPort> getAllByName(String name) throws UnknownHostException
{
return getAllByNameOverrideDefaults(name, null);
}

/**
*
* @param name Hostname + optional ports string
* @param port Port to connect on, overridden by values in hostname string, defaults to DatabaseDescriptor default if not specified anywhere.
*/
public static List<InetAddressAndPort> getAllByNameOverrideDefaults(String name, Integer port) throws UnknownHostException
{
HostAndPort hap = HostAndPort.fromString(name);
if (hap.hasPort())
{
port = hap.getPort();
}
Integer finalPort = port;

return Stream.of(InetAddress.getAllByName(hap.getHost()))
.map((address) -> getByAddressOverrideDefaults(address, finalPort))
.collect(Collectors.toList());
}

/**
*
* @param name Hostname + optional ports string
Expand Down
58 changes: 54 additions & 4 deletions src/java/org/apache/cassandra/locator/SimpleSeedProvider.java
Expand Up @@ -23,17 +23,31 @@
import java.util.List;
import java.util.Map;

import com.google.common.annotations.VisibleForTesting;

import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.FBUtilities;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SimpleSeedProvider implements SeedProvider
{
private static final Logger logger = LoggerFactory.getLogger(SimpleSeedProvider.class);
@VisibleForTesting
public static final String SEEDS_KEY = "seeds";
@VisibleForTesting
public static final String RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY = "resolve_multiple_ip_addresses_per_dns_record";
private static final int SEED_COUNT_WARN_THRESHOLD = CassandraRelevantProperties.SEED_COUNT_WARN_THRESHOLD.getInt(20);

private final String[] defaultSeeds;

public SimpleSeedProvider(Map<String, String> args) {}
public SimpleSeedProvider(Map<String, String> args)
{
defaultSeeds = new String[] { FBUtilities.getLocalAddressAndPort().getHostAddress(true) };
}

public List<InetAddressAndPort> getSeeds()
{
Expand All @@ -46,14 +60,44 @@ public List<InetAddressAndPort> getSeeds()
{
throw new AssertionError(e);
}
String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1);

assert conf.seed_provider != null : "conf.seed_provider is null!";

boolean resolveMultipleIps;
String[] hosts;

Map<String, String> parameters = conf.seed_provider.parameters;
if (parameters == null)
{
resolveMultipleIps = false;
hosts = defaultSeeds;
}
else
{
hosts = parameters.getOrDefault(SEEDS_KEY, defaultSeeds[0]).split(",", -1);
resolveMultipleIps = Boolean.parseBoolean(parameters.getOrDefault(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, Boolean.FALSE.toString()));
}

List<InetAddressAndPort> seeds = new ArrayList<>(hosts.length);

for (String host : hosts)
{
try
{
if(!host.trim().isEmpty()) {
seeds.add(InetAddressAndPort.getByName(host.trim()));
if (!host.trim().isEmpty())
{
if (resolveMultipleIps)
{
List<InetAddressAndPort> resolvedSeeds = InetAddressAndPort.getAllByName(host.trim());
seeds.addAll(resolvedSeeds);
logger.debug("{} resolves to {}", host, resolvedSeeds);
}
else
{
InetAddressAndPort addressAndPort = InetAddressAndPort.getByName(host.trim());
seeds.add(addressAndPort);
logger.debug("Only resolving one IP per DNS record - {} resolves to {}", host, addressAndPort);
}
}
}
catch (UnknownHostException ex)
Expand All @@ -62,6 +106,12 @@ public List<InetAddressAndPort> getSeeds()
logger.warn("Seed provider couldn't lookup host {}", host);
}
}

if (seeds.size() > SEED_COUNT_WARN_THRESHOLD)
logger.warn("Seed provider returned more than {} seeds. " +
"A large seed list may impact effectiveness of the third gossip round.",
SEED_COUNT_WARN_THRESHOLD);

return Collections.unmodifiableList(seeds);
}
}
176 changes: 176 additions & 0 deletions test/unit/org/apache/cassandra/locator/SimpleSeedProviderTest.java
@@ -0,0 +1,176 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.cassandra.locator;

import java.net.InetAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.junit.Test;

import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.utils.FBUtilities;
import org.mockito.MockedStatic;

import static java.lang.String.format;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.apache.cassandra.locator.SimpleSeedProvider.RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY;
import static org.apache.cassandra.locator.SimpleSeedProvider.SEEDS_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mockStatic;

public class SimpleSeedProviderTest
{
private static final String dnsName1 = "dns-name-1";
private static final String dnsName2 = "dns-name-2";

@Test
public void testSeedsResolution() throws Throwable
{
MockedStatic<InetAddressAndPort> inetAddressAndPortMock = null;
MockedStatic<DatabaseDescriptor> descriptorMock = null;
MockedStatic<FBUtilities> fbUtilitiesMock = null;

try
{
byte[] addressBytes1 = new byte[]{ 127, 0, 0, 1 };
byte[] addressBytes2 = new byte[]{ 127, 0, 0, 2 };
byte[] addressBytes3 = new byte[]{ 127, 0, 0, 3 };
InetAddressAndPort address1 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes1), addressBytes1, 7000);
InetAddressAndPort address2 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes2), addressBytes2, 7000);
InetAddressAndPort address3 = new InetAddressAndPort(InetAddress.getByAddress(addressBytes3), addressBytes3, 7000);

inetAddressAndPortMock = mockStatic(InetAddressAndPort.class);
inetAddressAndPortMock.when(() -> InetAddressAndPort.getByName(dnsName1)).thenReturn(address1);
inetAddressAndPortMock.when(() -> InetAddressAndPort.getAllByName(dnsName1)).thenReturn(singletonList(address1));
inetAddressAndPortMock.when(() -> InetAddressAndPort.getAllByName(dnsName2)).thenReturn(asList(address2, address3));
inetAddressAndPortMock.when(() -> InetAddressAndPort.getByName(dnsName2)).thenReturn(address2);

fbUtilitiesMock = mockStatic(FBUtilities.class);
fbUtilitiesMock.when(FBUtilities::getLocalAddressAndPort).thenReturn(address1);

descriptorMock = mockStatic(DatabaseDescriptor.class);

Map<String, String> seedProviderArgs = new HashMap<>();

//
// dns 1 without multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, dnsName1);
// resolve_multiple_ip_addresses_per_dns_record is implicitly false here

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
SimpleSeedProvider provider = new SimpleSeedProvider(null);
List<InetAddressAndPort> seeds = provider.getSeeds();

assertEquals(1, seeds.size());
assertEquals(address1, seeds.get(0));

//
// dns 2 without multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, dnsName2);
// resolve_multiple_ip_addresses_per_dns_record is implicitly false here

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
provider = new SimpleSeedProvider(null);
seeds = provider.getSeeds();

assertEquals(1, seeds.size());
assertTrue(seeds.contains(address2));

//
// dns 1 with multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, dnsName1);
seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
provider = new SimpleSeedProvider(null);
seeds = provider.getSeeds();

assertEquals(1, seeds.size());
assertEquals(address1, seeds.get(0));

//
// dns 2 with multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, dnsName2);
seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
provider = new SimpleSeedProvider(null);
seeds = provider.getSeeds();

assertEquals(2, seeds.size());
assertTrue(seeds.containsAll(asList(address2, address3)));

//
// dns 1 and dns 2 without multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, format("%s,%s", dnsName1, dnsName2));
seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "false");

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
provider = new SimpleSeedProvider(null);
seeds = provider.getSeeds();

assertEquals(2, seeds.size());
assertTrue(seeds.containsAll(asList(address1, address2)));

//
// dns 1 and dns 2 with multiple ips per record
//
seedProviderArgs.put(SEEDS_KEY, format("%s,%s", dnsName1, dnsName2));
seedProviderArgs.put(RESOLVE_MULTIPLE_IP_ADDRESSES_PER_DNS_RECORD_KEY, "true");

descriptorMock.when(DatabaseDescriptor::loadConfig).thenReturn(getConfig(seedProviderArgs));
provider = new SimpleSeedProvider(null);
seeds = provider.getSeeds();

assertEquals(3, seeds.size());
assertTrue(seeds.containsAll(asList(address1, address2, address3)));
}
finally
{
if (inetAddressAndPortMock != null)
inetAddressAndPortMock.close();

if (descriptorMock != null)
descriptorMock.close();

if (fbUtilitiesMock != null)
fbUtilitiesMock.close();
}
}

private static Config getConfig(Map<String, String> parameters)
{
Config config = new Config();
config.seed_provider = new ParameterizedClass();
config.seed_provider.class_name = SimpleSeedProvider.class.getName();
config.seed_provider.parameters = parameters;
return config;
}
}

0 comments on commit b07c312

Please sign in to comment.