Skip to content

Commit bd767e3

Browse files
dsmileydsmiley
authored
SOLR-17879: Fail to start if its major version is smaller than the cluster (#3510)
A Solr node will now fail to start if its major.minor version (e.g. 9.10) is *lower* than that of any existing Solr node in a SolrCloud cluster (as reported by info in "live_node").
1 parent fda4c40 commit bd767e3

File tree

6 files changed

+252
-16
lines changed

6 files changed

+252
-16
lines changed

‎solr/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,9 @@ Other Changes
266266
---------------------
267267
* SOLR-17620: SolrCloud "live_node" now has metadata: version of Solr, roles (Yuntong Qu, David Smiley)
268268

269+
* SOLR-17879: A Solr node will now fail to start if it's major.minor version (e.g. 9.10) is *lower* than that of any existing
270+
Solr node in a SolrCloud cluster (as reported by info in "live_node"). (David Smiley)
271+
269272
================== 9.9.1 ==================
270273
Bug Fixes
271274
---------------------

‎solr/core/src/java/org/apache/solr/cloud/ZkController.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,53 @@ private void checkNoOldClusterstate(final SolrZkClient zkClient) throws Interrup
613613
}
614614
}
615615

616+
/**
617+
* Checks version compatibility with other nodes in the cluster. Refuses to start if there's a
618+
* major.minor version difference between our Solr version and other nodes in the cluster. Note:
619+
* uses live nodes.
620+
*/
621+
private void checkClusterVersionCompatibility() throws InterruptedException, KeeperException {
622+
Optional<SolrVersion> lowestVersion = zkStateReader.fetchLowestSolrVersion();
623+
if (lowestVersion.isPresent()) {
624+
SolrVersion ourVersion = SolrVersion.LATEST;
625+
SolrVersion clusterVersion = lowestVersion.get();
626+
627+
if (ourVersion.lessThan(clusterVersion)) {
628+
log.warn(
629+
"Our Solr version {} is older than cluster version {}", ourVersion, clusterVersion);
630+
631+
if (EnvUtils.getPropertyAsBool("solr.cloud.downgrade.enabled", false)) {
632+
return;
633+
}
634+
635+
// Check major version compatibility
636+
if (ourVersion.getMajorVersion() < clusterVersion.getMajorVersion()) {
637+
String message =
638+
String.format(
639+
Locale.ROOT,
640+
"Refusing to start Solr, since our version is lower than the lowest version currently running in the cluster. "
641+
+ "Our version: %s, lowest version in cluster: %s.",
642+
ourVersion,
643+
clusterVersion);
644+
throw new SolrException(ErrorCode.INVALID_STATE, message);
645+
}
646+
647+
// Check minor version compatibility within the same major version
648+
if (ourVersion.getMajorVersion() == clusterVersion.getMajorVersion()
649+
&& ourVersion.getMinorVersion() < clusterVersion.getMinorVersion()) {
650+
String message =
651+
String.format(
652+
Locale.ROOT,
653+
"Refusing to start Solr, since our version is lower than the lowest version currently running in the cluster. "
654+
+ "Our version: %s, lowest version in cluster: %s.",
655+
ourVersion,
656+
clusterVersion);
657+
throw new SolrException(ErrorCode.INVALID_STATE, message);
658+
}
659+
}
660+
}
661+
}
662+
616663
public CloudSolrClient getSolrClient() {
617664
return getSolrCloudManager().getSolrClient();
618665
}
@@ -1033,6 +1080,7 @@ private void init() {
10331080

10341081
checkForExistingEphemeralNode();
10351082
registerLiveNodesListener();
1083+
checkClusterVersionCompatibility();
10361084

10371085
// Start the overseer now since the following code may need it's processing.
10381086
// Note: even when using distributed processing, we still create an Overseer anyway since

‎solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.solr.client.api.util.SolrVersion;
4242
import org.apache.solr.client.solrj.impl.Http2SolrClient;
4343
import org.apache.solr.common.MapWriter;
44+
import org.apache.solr.common.SolrException;
4445
import org.apache.solr.common.cloud.ClusterProperties;
4546
import org.apache.solr.common.cloud.ClusterState;
4647
import org.apache.solr.common.cloud.DocCollection;
@@ -66,6 +67,7 @@
6667
import org.apache.zookeeper.CreateMode;
6768
import org.apache.zookeeper.data.Stat;
6869
import org.hamcrest.Matchers;
70+
import org.junit.Ignore;
6971
import org.junit.Test;
7072

7173
@SolrTestCaseJ4.SuppressSSL
@@ -473,6 +475,155 @@ public void testTouchConfDir() throws Exception {
473475
}
474476
}
475477

478+
@Test
479+
@Ignore("Would need to disable ObjectReleaseTracker")
480+
public void testVersionCompatibilityFailsStartup() throws Exception {
481+
Path zkDir = createTempDir("testVersionCompatibilityFailsStartup");
482+
ZkTestServer server = new ZkTestServer(zkDir);
483+
try {
484+
server.run();
485+
486+
// Manually create a live node with a high version (99.0.0) to simulate
487+
// a newer cluster that the current node (SolrVersion.LATEST=10.0.0) cannot join
488+
try (SolrZkClient zkClient =
489+
new SolrZkClient.Builder()
490+
.withUrl(server.getZkAddress())
491+
.withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
492+
.build()) {
493+
494+
// Create cluster nodes first
495+
ZkController.createClusterZkNodes(zkClient);
496+
497+
String liveNodeName = "test_node:8983_solr";
498+
String liveNodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + liveNodeName;
499+
500+
// Create live node data with version 99.0.0
501+
Map<String, Object> liveNodeData =
502+
Map.of(LIVE_NODE_SOLR_VERSION, "99.0.0", LIVE_NODE_NODE_NAME, liveNodeName);
503+
byte[] data = Utils.toJSON(liveNodeData);
504+
505+
// persistent since we're about to close this zkClient
506+
zkClient.create(liveNodePath, data, CreateMode.PERSISTENT, true);
507+
}
508+
509+
// Now try to create a ZkController - this should fail due to version incompatibility
510+
CoreContainer cc = getCoreContainer();
511+
try {
512+
CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8984).build();
513+
514+
SolrException exception =
515+
expectThrows(
516+
SolrException.class,
517+
() -> {
518+
var zc = new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig);
519+
zc.close();
520+
});
521+
522+
// Verify the exception is due to version incompatibility
523+
assertEquals(
524+
"Expected INVALID_STATE error code",
525+
SolrException.ErrorCode.INVALID_STATE.code,
526+
exception.code());
527+
assertTrue(
528+
"Exception message should mention refusing to start: " + exception.getMessage(),
529+
exception.getMessage().contains("Refusing to start Solr"));
530+
assertTrue(
531+
"Exception message should mention minor version: " + exception.getMessage(),
532+
exception.getMessage().contains("minor version"));
533+
assertTrue(
534+
"Exception message should mention our version: " + exception.getMessage(),
535+
exception.getMessage().contains("10.0.0"));
536+
assertTrue(
537+
"Exception message should mention cluster version: " + exception.getMessage(),
538+
exception.getMessage().contains("99.0.0"));
539+
} finally {
540+
cc.shutdown();
541+
}
542+
} finally {
543+
server.shutdown();
544+
}
545+
}
546+
547+
@Ignore("Would need to disable ObjectReleaseTracker")
548+
public void testMinorVersionCompatibilityFailsStartup() throws Exception {
549+
Path zkDir = createTempDir("testMinorVersionCompatibilityFailsStartup");
550+
ZkTestServer server = new ZkTestServer(zkDir);
551+
try {
552+
server.run();
553+
554+
// Create a higher minor version based on SolrVersion.LATEST for cluster simulation
555+
SolrVersion currentVersion = SolrVersion.LATEST;
556+
SolrVersion higherMinorVersion =
557+
SolrVersion.forIntegers(
558+
currentVersion.getMajorVersion(),
559+
currentVersion.getMinorVersion() + 1,
560+
currentVersion.getPatchVersion());
561+
562+
// Manually create a live node with a higher minor version to simulate
563+
// a newer cluster that the current node (SolrVersion.LATEST) cannot join
564+
try (SolrZkClient zkClient =
565+
new SolrZkClient.Builder()
566+
.withUrl(server.getZkAddress())
567+
.withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
568+
.build()) {
569+
570+
// Create cluster nodes first
571+
ZkController.createClusterZkNodes(zkClient);
572+
573+
String liveNodeName = "test_node:8983_solr";
574+
String liveNodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + liveNodeName;
575+
576+
// Create live node data with higher minor version (same major, higher minor than LATEST)
577+
Map<String, Object> liveNodeData =
578+
Map.of(
579+
LIVE_NODE_SOLR_VERSION,
580+
higherMinorVersion.toString(),
581+
LIVE_NODE_NODE_NAME,
582+
liveNodeName);
583+
byte[] data = Utils.toJSON(liveNodeData);
584+
585+
// persistent since we're about to close this zkClient
586+
zkClient.create(liveNodePath, data, CreateMode.PERSISTENT, true);
587+
}
588+
589+
// Now try to create a ZkController - this should fail due to minor version incompatibility
590+
CoreContainer cc = getCoreContainer();
591+
try {
592+
CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8984).build();
593+
594+
SolrException exception =
595+
expectThrows(
596+
SolrException.class,
597+
() -> {
598+
var zc = new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig);
599+
zc.close();
600+
});
601+
602+
// Verify the exception is due to minor version incompatibility
603+
assertEquals(
604+
"Expected INVALID_STATE error code",
605+
SolrException.ErrorCode.INVALID_STATE.code,
606+
exception.code());
607+
assertTrue(
608+
"Exception message should mention refusing to start: " + exception.getMessage(),
609+
exception.getMessage().contains("Refusing to start Solr"));
610+
assertTrue(
611+
"Exception message should mention minor version: " + exception.getMessage(),
612+
exception.getMessage().contains("minor version"));
613+
assertTrue(
614+
"Exception message should mention our version: " + exception.getMessage(),
615+
exception.getMessage().contains(currentVersion.toString()));
616+
assertTrue(
617+
"Exception message should mention cluster version: " + exception.getMessage(),
618+
exception.getMessage().contains(higherMinorVersion.toString()));
619+
} finally {
620+
cc.shutdown();
621+
}
622+
} finally {
623+
server.shutdown();
624+
}
625+
}
626+
476627
public void testCheckNoOldClusterstate() throws Exception {
477628
Path zkDir = createTempDir("testCheckNoOldClusterstate");
478629
ZkTestServer server = new ZkTestServer(zkDir);

‎solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -923,8 +923,9 @@ public void testFetchLowestSolrVersion_validNodes() throws Exception {
923923
zkClient.create(livePath + "/" + node2, data2, CreateMode.EPHEMERAL, true);
924924

925925
var lowestVersion = reader.fetchLowestSolrVersion();
926+
assertTrue("Expected lowest version to be present", lowestVersion.isPresent());
926927
assertEquals(
927-
"Expected lowest version to be 9.0.1", SolrVersion.valueOf("9.0.1"), lowestVersion);
928+
"Expected lowest version to be 9.0.1", SolrVersion.valueOf("9.0.1"), lowestVersion.get());
928929
}
929930

930931
/** Test when the only live node has empty data. */
@@ -943,7 +944,9 @@ public void testFetchLowestSolrVersion_noData() throws Exception {
943944
String emptyNode = "empty_node";
944945
zkClient.create(livePath + "/" + emptyNode, new byte[0], CreateMode.EPHEMERAL, true);
945946

946-
assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), reader.fetchLowestSolrVersion());
947+
var lowestVersion = reader.fetchLowestSolrVersion();
948+
assertTrue("Expected lowest version to be present for empty node", lowestVersion.isPresent());
949+
assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), lowestVersion.get());
947950
}
948951

949952
/** Test when two live nodes exist; one is blank and the other has a high version */
@@ -965,11 +968,32 @@ public void testFetchLowestSolrVersion_blankAndHighVersion() throws Exception {
965968
CreateMode.EPHEMERAL,
966969
true);
967970

968-
assertEquals("after high node", SolrVersion.LATEST, reader.fetchLowestSolrVersion());
971+
var lowestVersion1 = reader.fetchLowestSolrVersion();
972+
assertTrue(
973+
"Expected lowest version to be present for high version node", lowestVersion1.isPresent());
974+
assertEquals("after high node", SolrVersion.valueOf("888.0.0"), lowestVersion1.get());
969975

970976
String node2 = "node2_solr";
971977
zkClient.create(livePath + "/" + node2, new byte[0], CreateMode.EPHEMERAL, true);
972978

973-
assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), reader.fetchLowestSolrVersion());
979+
var lowestVersion2 = reader.fetchLowestSolrVersion();
980+
assertTrue("Expected lowest version to be present for empty node", lowestVersion2.isPresent());
981+
assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), lowestVersion2.get());
982+
}
983+
984+
/** Test when no live nodes exist - should return empty Optional */
985+
public void testFetchLowestSolrVersion_noLiveNodes() throws Exception {
986+
SolrZkClient zkClient = fixture.zkClient;
987+
ZkStateReader reader = fixture.reader;
988+
String livePath = ZkStateReader.LIVE_NODES_ZKNODE;
989+
990+
// Clear any existing live node children.
991+
List<String> nodes = zkClient.getChildren(livePath, null, true);
992+
for (String node : nodes) {
993+
zkClient.delete(livePath + "/" + node, -1, true);
994+
}
995+
996+
var lowestVersion = reader.fetchLowestSolrVersion();
997+
assertFalse("Expected no lowest version when no live nodes exist", lowestVersion.isPresent());
974998
}
975999
}

‎solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@ Before starting an upgrade to this version of Solr, please take the time to revi
2626

2727
// TODO add similar text that previous releases have at this spot.
2828

29+
There is a new limitation introduced in Solr 9.10, and that which is especially relevant to Solr 10
30+
and beyond.
31+
A Solr node will now fail to start if it's major.minor version (e.g. 9.10) is *lower* than that of any existing Solr node in a SolrCloud cluster (as reported by info in "live_node").
32+
What this
33+
means is that Solr supports rolling _upgrades_ but not rolling _downgrades_ spanning a major version.
34+
This compatibility safeguard can be toggled with "SOLR_CLOUD_DOWNGRADE_ENABLED".
35+
2936
== System Requirements
3037

3138
Minimum Java version for Solr 10.x is Java 21.
@@ -49,7 +56,7 @@ Now you have to explicitly provide a `--delete-config` option to delete the con
4956

5057
The `bin/solr start --bootstrap_conf` flag is a legacy feature for converting from Solr to SolrCloud mode and has been removed.
5158

52-
The system property "bootstrap_confdir" (recently renamed to "solr.configset.bootstrap.confdir") used in `bin/solr start` allwowed a collection creation command to default to the examination of system properties in the absence of proper creation parameters by the same name (like "collection.configName").
59+
The system property "bootstrap_confdir" (recently renamed to "solr.configset.bootstrap.confdir") used in `bin/solr start` allwowed a collection creation command to default to the examination of system properties in the absence of proper creation parameters by the same name (like "collection.configName").
5360
That no longer happens in Solr 10. It is only used to load a configuration as a ConfigSet.
5461

5562
=== SolrJ

‎solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.Map;
3232
import java.util.Map.Entry;
3333
import java.util.Objects;
34+
import java.util.Optional;
3435
import java.util.Set;
3536
import java.util.SortedSet;
3637
import java.util.TreeSet;
@@ -875,38 +876,40 @@ public void removeLiveNodesListener(LiveNodesListener listener) {
875876
}
876877

877878
/**
878-
* Returns the lowest Solr version among all live nodes in the cluster. It's not greater than
879-
* {@link SolrVersion#LATEST_STRING}. Will not return null. If older Solr nodes have joined that
880-
* don't declare their version, the result won't be accurate, but it's at least an upper bound on
881-
* the possible version it might be.
879+
* Returns the lowest Solr version among all live nodes in the cluster. If older Solr nodes have
880+
* joined that don't declare their version, the result won't be accurate, but it's at least an
881+
* upper bound on the possible version it might be.
882882
*
883-
* @return the lowest Solr version of the cluster; not null
883+
* @return an Optional containing the lowest Solr version of nodes in the cluster, or empty if no
884+
* live nodes exist or all nodes return 9.9.0 for unspecified versions
884885
*/
885-
public SolrVersion fetchLowestSolrVersion() throws KeeperException, InterruptedException {
886+
public Optional<SolrVersion> fetchLowestSolrVersion()
887+
throws KeeperException, InterruptedException {
886888
List<String> liveNodeNames = zkClient.getChildren(LIVE_NODES_ZKNODE, null, true);
887-
SolrVersion lowest = SolrVersion.LATEST; // current software
889+
SolrVersion lowest = null;
888890
// the last version to not specify its version in live nodes
889891
final SolrVersion UNSPECIFIED_VERSION = SolrVersion.valueOf("9.9.0");
892+
890893
for (String nodeName : liveNodeNames) {
891894
String path = LIVE_NODES_ZKNODE + "/" + nodeName;
892895
byte[] data = zkClient.getData(path, null, null, true);
893896
if (data == null || data.length == 0) {
894-
return UNSPECIFIED_VERSION;
897+
return Optional.of(UNSPECIFIED_VERSION);
895898
}
896899

897900
@SuppressWarnings("unchecked")
898901
Map<String, Object> props = (Map<String, Object>) Utils.fromJSON(data);
899902
String nodeVersionStr = (String) props.get(LIVE_NODE_SOLR_VERSION);
900903
if (nodeVersionStr == null) { // weird
901904
log.warn("No Solr version found: {}", props);
902-
return UNSPECIFIED_VERSION;
905+
return Optional.of(UNSPECIFIED_VERSION);
903906
}
904907
SolrVersion nodeVersion = SolrVersion.valueOf(nodeVersionStr);
905-
if (nodeVersion.compareTo(lowest) < 0) {
908+
if (lowest == null || nodeVersion.compareTo(lowest) < 0) {
906909
lowest = nodeVersion;
907910
}
908911
}
909-
return lowest;
912+
return Optional.ofNullable(lowest);
910913
}
911914

912915
/**

0 commit comments

Comments
 (0)