Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-16507: Change SplitShardCmd to not use NodeStateProvider #1485

Merged
merged 4 commits into from Mar 31, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -17,7 +17,6 @@

package org.apache.solr.cloud.api.collections;

import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable.CORE_IDX;
import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.RANDOM;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
Expand All @@ -42,11 +41,13 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.VersionedData;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;
Expand All @@ -65,7 +66,6 @@
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CommonParams;
Expand Down Expand Up @@ -873,42 +873,85 @@ public static void checkDiskSpace(
SolrIndexSplitter.SplitMethod method,
SolrCloudManager cloudManager)
throws SolrException {

// check that enough disk space is available on the parent leader node
// otherwise the actual index splitting will always fail
NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
Map<String, Object> nodeValues =
nodeStateProvider.getNodeValues(
parentShardLeader.getNodeName(), Collections.singletonList(ImplicitSnitch.DISK));
Map<String, Map<String, List<Replica>>> infos =
nodeStateProvider.getReplicaInfo(
parentShardLeader.getNodeName(), Collections.singletonList(CORE_IDX.metricsAttribute));
if (infos.get(collection) == null || infos.get(collection).get(shard) == null) {

ModifiableSolrParams params;
String metricName;
GenericSolrRequest req;
SolrResponse rsp;
dsmiley marked this conversation as resolved.
Show resolved Hide resolved

metricName =
new StringBuilder("solr.core.")
dsmiley marked this conversation as resolved.
Show resolved Hide resolved
.append(collection)
.append(".")
.append(shard)
.append(".")
.append(Utils.parseMetricsReplicaName(collection, parentShardLeader.getCoreName()))
.append(":INDEX.sizeInBytes")
.toString();

params = new ModifiableSolrParams();
params.add("key", metricName);

req = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics", params);
dsmiley marked this conversation as resolved.
Show resolved Hide resolved
try {
rsp = cloudManager.request(req);
} catch (Exception e) {
log.error("Error occurred while checking the disk space of the node");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

always propagate the exception!

return;
}

if (rsp.getResponse() == null) {
log.warn("cannot verify information for parent shard leader");
return;
}
// find the leader
List<Replica> lst = infos.get(collection).get(shard);
Double indexSize = null;
for (Replica info : lst) {
if (info.getCoreName().equals(parentShardLeader.getCoreName())) {
Number size = (Number) info.get(CORE_IDX.metricsAttribute);
if (size == null) {
log.warn("cannot verify information for parent shard leader");
return;
}
indexSize = (Double) CORE_IDX.convertVal(size);
break;
}

NamedList<Object> response = rsp.getResponse();
Object value = response.findRecursive("metrics", metricName);
if (value == null) {
log.warn("cannot verify information for parent shard leader");
return;
}

dsmiley marked this conversation as resolved.
Show resolved Hide resolved
Number size = (Number) value;
if (size == null) {
log.warn("cannot verify information for parent shard leader");
return;
}
if (indexSize == null) {
log.warn("missing replica information for parent shard leader");
double indexSize = size.doubleValue();

metricName = "solr.node:CONTAINER.fs.usableSpace";
params = new ModifiableSolrParams();
params.add("key", metricName);
dsmiley marked this conversation as resolved.
Show resolved Hide resolved

req = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics", params);
dsmiley marked this conversation as resolved.
Show resolved Hide resolved
try {
rsp = cloudManager.request(req);
} catch (Exception e) {
log.error("Error occurred while checking the disk space of the node");
return;
}
Number freeSize = (Number) nodeValues.get(ImplicitSnitch.DISK);

if (rsp.getResponse() == null) {
log.warn("cannot verify information for parent shard leader");
return;
}

response = rsp.getResponse();
value = response.findRecursive("metrics", metricName);
if (value == null) {
log.warn("cannot verify information for parent shard leader");
return;
}

Number freeSize = (Number) value;
if (freeSize == null) {
log.warn("missing node disk space information for parent shard leader");
return;
}

// 100% more for REWRITE, 5% more for LINK
double neededSpace =
method == SolrIndexSplitter.SplitMethod.REWRITE ? 2.0 * indexSize : 1.05 * indexSize;
Expand Down