Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#7318, discover primaryDataSourceName automatically instead of write into rule config #8901

Merged
merged 7 commits into from Jan 5, 2021
Expand Up @@ -37,5 +37,5 @@ public final class HARuleConfiguration implements RuleConfiguration {

private final Map<String, ShardingSphereAlgorithmConfiguration> loadBalancers;

private final ShardingSphereAlgorithmConfiguration haType;
private final ShardingSphereAlgorithmConfiguration haConfiguration;
}
Expand Up @@ -17,25 +17,21 @@

package org.apache.shardingsphere.ha.api.config.rule;

import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.RequiredArgsConstructor;

import java.util.List;

/**
* HA data source rule configuration.
*/
@AllArgsConstructor
@RequiredArgsConstructor
@Getter
public final class HADataSourceRuleConfiguration {

private final String name;

@Setter
private String primaryDataSourceName;

private final List<String> replicaDataSourceNames;
private final List<String> dataSourceNames;

private final String loadBalancerName;

Expand Down
Expand Up @@ -57,4 +57,12 @@ public interface HAType extends TypedSPI {
* Stop periodical update.
*/
void stopPeriodicalUpdate();

/**
* Get primary data source.
*
* @return primary data source
*/
String getPrimaryDataSource();

}
Expand Up @@ -31,8 +31,8 @@ public interface ReplicaLoadBalanceAlgorithm extends ShardingSphereAlgorithm {
*
* @param name HA logic data source name
* @param primaryDataSourceName name of primary data sources
* @param replicaDataSourceNames names of replica data sources
* @param dataSourceNames names of replica data sources
* @return name of selected data source
*/
String getDataSource(String name, String primaryDataSourceName, List<String> replicaDataSourceNames);
String getDataSource(String name, String primaryDataSourceName, List<String> dataSourceNames);
}
Expand Up @@ -35,8 +35,8 @@ public final class RandomReplicaLoadBalanceAlgorithm implements ReplicaLoadBalan
private Properties props = new Properties();

@Override
public String getDataSource(final String name, final String primaryDataSourceName, final List<String> replicaDataSourceNames) {
return replicaDataSourceNames.get(ThreadLocalRandom.current().nextInt(replicaDataSourceNames.size()));
public String getDataSource(final String name, final String primaryDataSourceName, final List<String> dataSourceNames) {
return dataSourceNames.get(ThreadLocalRandom.current().nextInt(dataSourceNames.size()));
}

@Override
Expand Down
Expand Up @@ -31,11 +31,11 @@ public final class RoundRobinReplicaLoadBalanceAlgorithm implements ReplicaLoadB
private static final ConcurrentHashMap<String, AtomicInteger> COUNTS = new ConcurrentHashMap<>();

@Override
public String getDataSource(final String name, final String primaryDataSourceName, final List<String> replicaDataSourceNames) {
public String getDataSource(final String name, final String primaryDataSourceName, final List<String> dataSourceNames) {
AtomicInteger count = COUNTS.containsKey(name) ? COUNTS.get(name) : new AtomicInteger(0);
COUNTS.putIfAbsent(name, count);
count.compareAndSet(replicaDataSourceNames.size(), 0);
return replicaDataSourceNames.get(Math.abs(count.getAndIncrement()) % replicaDataSourceNames.size());
count.compareAndSet(dataSourceNames.size(), 0);
return dataSourceNames.get(Math.abs(count.getAndIncrement()) % dataSourceNames.size());
}

@Override
Expand Down
Expand Up @@ -40,9 +40,7 @@ public final class HADataSourceRule {

private final String name;

private final String primaryDataSourceName;

private final List<String> replicaDataSourceNames;
private final List<String> dataSourceNames;

private final ReplicaLoadBalanceAlgorithm loadBalancer;

Expand All @@ -54,25 +52,23 @@ public final class HADataSourceRule {
public HADataSourceRule(final HADataSourceRuleConfiguration config, final ReplicaLoadBalanceAlgorithm loadBalancer) {
checkConfiguration(config);
name = config.getName();
primaryDataSourceName = config.getPrimaryDataSourceName();
replicaDataSourceNames = config.getReplicaDataSourceNames();
dataSourceNames = config.getDataSourceNames();
this.loadBalancer = loadBalancer;
this.replicaQuery = config.isReplicaQuery();
}

private void checkConfiguration(final HADataSourceRuleConfiguration config) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(config.getName()), "Name is required.");
Preconditions.checkArgument(!Strings.isNullOrEmpty(config.getPrimaryDataSourceName()), "Primary data source name is required.");
Preconditions.checkArgument(null != config.getReplicaDataSourceNames() && !config.getReplicaDataSourceNames().isEmpty(), "Replica data source names are required.");
Preconditions.checkArgument(null != config.getDataSourceNames() && !config.getDataSourceNames().isEmpty(), "Replica data source names are required.");
}

/**
* Get replica data source names.
*
* @return available replica data source names
*/
public List<String> getReplicaDataSourceNames() {
return replicaDataSourceNames.stream().filter(each -> !disabledDataSourceNames.contains(each)).collect(Collectors.toList());
public List<String> getDataSourceNames() {
return dataSourceNames.stream().filter(each -> !disabledDataSourceNames.contains(each)).collect(Collectors.toList());
}

/**
Expand All @@ -97,8 +93,7 @@ public void updateDisabledDataSourceNames(final String dataSourceName, final boo
public Map<String, Collection<String>> getDataSourceMapper() {
Map<String, Collection<String>> result = new HashMap<>(1, 1);
Collection<String> actualDataSourceNames = new LinkedList<>();
actualDataSourceNames.add(primaryDataSourceName);
actualDataSourceNames.addAll(replicaDataSourceNames);
actualDataSourceNames.addAll(dataSourceNames);
result.put(name, actualDataSourceNames);
return result;
}
Expand Down
Expand Up @@ -19,6 +19,7 @@

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import lombok.Getter;
import org.apache.shardingsphere.ha.spi.HAType;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmFactory;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
Expand Down Expand Up @@ -53,6 +54,7 @@ public final class HARule implements DataSourceContainedRule, StatusContainedRul
ShardingSphereServiceLoader.register(HAType.class);
}

@Getter
private static HAType haType;

private final Map<String, ReplicaLoadBalanceAlgorithm> loadBalancers = new LinkedHashMap<>();
Expand All @@ -72,7 +74,7 @@ public HARule(final HARuleConfiguration config, final DatabaseType databaseType,
dataSourceRules.put(each.getName(), new HADataSourceRule(each, loadBalanceAlgorithm));
}
if (null == haType) {
haType = TypedSPIRegistry.getRegisteredService(HAType.class, config.getHaType().getType(), config.getHaType().getProps());
haType = TypedSPIRegistry.getRegisteredService(HAType.class, config.getHaConfiguration().getType(), config.getHaConfiguration().getProps());
haType.updatePrimaryDataSource(dataSourceMap, schemaName);
} else {
haType.stopPeriodicalUpdate();
Expand Down
Expand Up @@ -47,8 +47,7 @@ public final class HARuleBuilder implements ShardingSphereRuleBuilder<HARule, HA
public HARule build(final HARuleConfiguration ruleConfig) {
Set<String> dataSourceSet = new HashSet<>(128, 1);
for (HADataSourceRuleConfiguration each : ruleConfig.getDataSources()) {
dataSourceSet.add(each.getPrimaryDataSourceName());
dataSourceSet.addAll(each.getReplicaDataSourceNames());
dataSourceSet.addAll(each.getDataSourceNames());
}
dataSourceMap.entrySet().removeIf(stringDataSourceEntry -> !dataSourceSet.contains(stringDataSourceEntry.getKey()));
return new HARule(ruleConfig, databaseType, dataSourceMap, schemaName);
Expand Down
Expand Up @@ -34,9 +34,7 @@ public final class YamlHADataSourceRuleConfiguration implements YamlConfiguratio

private String name;

private String primaryDataSourceName;

private List<String> replicaDataSourceNames = new ArrayList<>();
private List<String> dataSourceNames = new ArrayList<>();

private String loadBalancerName;

Expand Down
Expand Up @@ -51,8 +51,7 @@ public YamlHARuleConfiguration swapToYamlConfiguration(final AlgorithmProvidedHA
private YamlHADataSourceRuleConfiguration swapToYamlConfiguration(final HADataSourceRuleConfiguration dataSourceRuleConfig) {
YamlHADataSourceRuleConfiguration result = new YamlHADataSourceRuleConfiguration();
result.setName(dataSourceRuleConfig.getName());
result.setPrimaryDataSourceName(dataSourceRuleConfig.getPrimaryDataSourceName());
result.setReplicaDataSourceNames(dataSourceRuleConfig.getReplicaDataSourceNames());
result.setDataSourceNames(dataSourceRuleConfig.getDataSourceNames());
result.setLoadBalancerName(dataSourceRuleConfig.getLoadBalancerName());
result.setReplicaQuery(dataSourceRuleConfig.isReplicaQuery());
return result;
Expand All @@ -70,7 +69,7 @@ public AlgorithmProvidedHARuleConfiguration swapToObject(final YamlHARuleConfigu
}

private HADataSourceRuleConfiguration swapToObject(final String name, final YamlHADataSourceRuleConfiguration yamlDataSourceRuleConfig) {
return new HADataSourceRuleConfiguration(name, yamlDataSourceRuleConfig.getPrimaryDataSourceName(), yamlDataSourceRuleConfig.getReplicaDataSourceNames(),
return new HADataSourceRuleConfiguration(name, yamlDataSourceRuleConfig.getDataSourceNames(),
yamlDataSourceRuleConfig.getLoadBalancerName(), yamlDataSourceRuleConfig.isReplicaQuery());
}

Expand Down
Expand Up @@ -49,17 +49,16 @@ public YamlHARuleConfiguration swapToYamlConfiguration(final HARuleConfiguration
if (null != data.getLoadBalancers()) {
data.getLoadBalancers().forEach((key, value) -> result.getLoadBalancers().put(key, algorithmSwapper.swapToYamlConfiguration(value)));
}
if (null != data.getHaType()) {
result.setHaType(algorithmSwapper.swapToYamlConfiguration(data.getHaType()));
if (null != data.getHaConfiguration()) {
result.setHaType(algorithmSwapper.swapToYamlConfiguration(data.getHaConfiguration()));
}
return result;
}

private YamlHADataSourceRuleConfiguration swapToYamlConfiguration(final HADataSourceRuleConfiguration dataSourceRuleConfig) {
YamlHADataSourceRuleConfiguration result = new YamlHADataSourceRuleConfiguration();
result.setName(dataSourceRuleConfig.getName());
result.setPrimaryDataSourceName(dataSourceRuleConfig.getPrimaryDataSourceName());
result.setReplicaDataSourceNames(dataSourceRuleConfig.getReplicaDataSourceNames());
result.setDataSourceNames(dataSourceRuleConfig.getDataSourceNames());
result.setLoadBalancerName(dataSourceRuleConfig.getLoadBalancerName());
result.setReplicaQuery(dataSourceRuleConfig.isReplicaQuery());
return result;
Expand All @@ -80,7 +79,7 @@ public HARuleConfiguration swapToObject(final YamlHARuleConfiguration yamlConfig
}

private HADataSourceRuleConfiguration swapToObject(final String name, final YamlHADataSourceRuleConfiguration yamlDataSourceRuleConfig) {
return new HADataSourceRuleConfiguration(name, yamlDataSourceRuleConfig.getPrimaryDataSourceName(), yamlDataSourceRuleConfig.getReplicaDataSourceNames(),
return new HADataSourceRuleConfiguration(name, yamlDataSourceRuleConfig.getDataSourceNames(),
yamlDataSourceRuleConfig.getLoadBalancerName(), yamlDataSourceRuleConfig.isReplicaQuery());
}

Expand Down
Expand Up @@ -33,9 +33,9 @@ public void assertGetDataSource() {
String primaryDataSourceName = "test_primary_ds";
String replicaDataSourceName1 = "test_replica_ds_1";
String replicaDataSourceName2 = "test_replica_ds_2";
List<String> replicaDataSourceNames = Arrays.asList(replicaDataSourceName1, replicaDataSourceName2);
assertTrue(replicaDataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames)));
assertTrue(replicaDataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames)));
assertTrue(replicaDataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames)));
List<String> dataSourceNames = Arrays.asList(replicaDataSourceName1, replicaDataSourceName2);
assertTrue(dataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames)));
assertTrue(dataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames)));
assertTrue(dataSourceNames.contains(randomReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames)));
}
}
Expand Up @@ -34,9 +34,9 @@ public void assertGetDataSource() {
String primaryDataSourceName = "test_primary_ds";
String replicaDataSourceName1 = "test_replica_ds_1";
String replicaDataSourceName2 = "test_replica_ds_2";
List<String> replicaDataSourceNames = Arrays.asList(replicaDataSourceName1, replicaDataSourceName2);
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames), is(replicaDataSourceName1));
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames), is(replicaDataSourceName2));
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, replicaDataSourceNames), is(replicaDataSourceName1));
List<String> dataSourceNames = Arrays.asList(replicaDataSourceName1, replicaDataSourceName2);
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames), is(replicaDataSourceName1));
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames), is(replicaDataSourceName2));
assertThat(roundRobinReplicaLoadBalanceAlgorithm.getDataSource("ds", primaryDataSourceName, dataSourceNames), is(replicaDataSourceName1));
}
}
Expand Up @@ -43,6 +43,11 @@ public void startPeriodicalUpdate(final Map<String, DataSource> dataSourceMap, f
public void stopPeriodicalUpdate() {
}

@Override
public String getPrimaryDataSource() {
return null;
}

@Override
public String getType() {
return "Test";
Expand Down
Expand Up @@ -34,56 +34,51 @@
public final class HADataSourceRuleTest {

private final HADataSourceRule haDataSourceRule = new HADataSourceRule(
new HADataSourceRuleConfiguration("test_pr", "primary_ds", Arrays.asList("replica_ds_0", "replica_ds_1"), "random", true), new RandomReplicaLoadBalanceAlgorithm());
new HADataSourceRuleConfiguration("test_pr", Arrays.asList("replica_ds_0", "replica_ds_1"), "random", true), new RandomReplicaLoadBalanceAlgorithm());

@Test(expected = IllegalArgumentException.class)
public void assertNewHADataSourceRuleWithoutName() {
new HADataSourceRule(new HADataSourceRuleConfiguration("", "primary_ds", Collections.singletonList("replica_ds"), null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
}

@Test(expected = IllegalArgumentException.class)
public void assertNewHADataSourceRuleWithoutPrimaryDataSourceName() {
new HADataSourceRule(new HADataSourceRuleConfiguration("ds", "", Collections.singletonList("replica_ds"), null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
new HADataSourceRule(new HADataSourceRuleConfiguration("", Collections.singletonList("replica_ds"), null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
}

@Test(expected = IllegalArgumentException.class)
public void assertNewHADataSourceRuleWithNullReplicaDataSourceName() {
new HADataSourceRule(new HADataSourceRuleConfiguration("ds", "primary_ds", null, null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
new HADataSourceRule(new HADataSourceRuleConfiguration("ds", null, null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
}

@Test(expected = IllegalArgumentException.class)
public void assertNewHADataSourceRuleWithEmptyReplicaDataSourceName() {
new HADataSourceRule(new HADataSourceRuleConfiguration("ds", "primary_ds", Collections.emptyList(), null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
new HADataSourceRule(new HADataSourceRuleConfiguration("ds", Collections.emptyList(), null, true), new RoundRobinReplicaLoadBalanceAlgorithm());
}

@Test
public void assertGetReplicaDataSourceNamesWithoutDisabledDataSourceNames() {
assertThat(haDataSourceRule.getReplicaDataSourceNames(), is(Arrays.asList("replica_ds_0", "replica_ds_1")));
public void assertGetDataSourceNamesWithoutDisabledDataSourceNames() {
assertThat(haDataSourceRule.getDataSourceNames(), is(Arrays.asList("replica_ds_0", "replica_ds_1")));
}

@Test
public void assertGetReplicaDataSourceNamesWithDisabledDataSourceNames() {
public void assertGetDataSourceNamesWithDisabledDataSourceNames() {
haDataSourceRule.updateDisabledDataSourceNames("replica_ds_0", true);
assertThat(haDataSourceRule.getReplicaDataSourceNames(), is(Collections.singletonList("replica_ds_1")));
assertThat(haDataSourceRule.getDataSourceNames(), is(Collections.singletonList("replica_ds_1")));
}

@Test
public void assertUpdateDisabledDataSourceNamesForDisabled() {
haDataSourceRule.updateDisabledDataSourceNames("replica_ds_0", true);
assertThat(haDataSourceRule.getReplicaDataSourceNames(), is(Collections.singletonList("replica_ds_1")));
assertThat(haDataSourceRule.getDataSourceNames(), is(Collections.singletonList("replica_ds_1")));
}

@Test
public void assertUpdateDisabledDataSourceNamesForEnabled() {
haDataSourceRule.updateDisabledDataSourceNames("replica_ds_0", true);
haDataSourceRule.updateDisabledDataSourceNames("replica_ds_0", false);
assertThat(haDataSourceRule.getReplicaDataSourceNames(), is(Arrays.asList("replica_ds_0", "replica_ds_1")));
assertThat(haDataSourceRule.getDataSourceNames(), is(Arrays.asList("replica_ds_0", "replica_ds_1")));
}

@Test
public void assertGetDataSourceMapper() {
Map<String, Collection<String>> actual = haDataSourceRule.getDataSourceMapper();
Map<String, Collection<String>> expected = ImmutableMap.of("test_pr", Arrays.asList("primary_ds", "replica_ds_0", "replica_ds_1"));
Map<String, Collection<String>> expected = ImmutableMap.of("test_pr", Arrays.asList("replica_ds_0", "replica_ds_1"));
assertThat(actual, is(expected));
}
}