Skip to content

Commit

Permalink
SOLR-16809: Converge logic for hidden sysProps
Browse files Browse the repository at this point in the history
  • Loading branch information
HoustonPutman committed Jul 5, 2023
1 parent 9efb368 commit 98c1988
Show file tree
Hide file tree
Showing 16 changed files with 196 additions and 182 deletions.
2 changes: 2 additions & 0 deletions solr/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,8 @@ Bug Fixes

* SOLR-16619: Fix solr scripts running on IBM i (Jesse Gorzinski via Eric Pugh)

* SOLR-16809: The configuration for hiding sensitive sysProp information has been joined under `-Dsolr.hiddenSysProps` and `SOLR_HIDDEN_SYS_PROPS`. (Houston Putman, David Smiley)

Dependency Upgrades
---------------------
* PR#1494: Upgrade forbiddenapis to 3.5 (Uwe Schindler)
Expand Down
23 changes: 0 additions & 23 deletions solr/core/src/java/org/apache/solr/core/MetricsConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,11 @@
package org.apache.solr.core;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

/** */
public class MetricsConfig {

private final PluginInfo[] metricReporters;
private final Set<String> hiddenSysProps;
private final PluginInfo counterSupplier;
private final PluginInfo meterSupplier;
private final PluginInfo timerSupplier;
Expand All @@ -39,7 +36,6 @@ public class MetricsConfig {
private MetricsConfig(
boolean enabled,
PluginInfo[] metricReporters,
Set<String> hiddenSysProps,
PluginInfo counterSupplier,
PluginInfo meterSupplier,
PluginInfo timerSupplier,
Expand All @@ -51,7 +47,6 @@ private MetricsConfig(
CacheConfig cacheConfig) {
this.enabled = enabled;
this.metricReporters = metricReporters;
this.hiddenSysProps = hiddenSysProps;
this.counterSupplier = counterSupplier;
this.meterSupplier = meterSupplier;
this.timerSupplier = timerSupplier;
Expand Down Expand Up @@ -97,14 +92,6 @@ public Object getNullObject() {
return nullObject;
}

public Set<String> getHiddenSysProps() {
if (enabled) {
return hiddenSysProps;
} else {
return Collections.emptySet();
}
}

/** Symbolic name to use as plugin class name for no-op implementations. */
public static final String NOOP_IMPL_CLASS = "__noop__";

Expand Down Expand Up @@ -145,7 +132,6 @@ public PluginInfo getHistogramSupplier() {

public static class MetricsConfigBuilder {
private PluginInfo[] metricReporterPlugins = new PluginInfo[0];
private Set<String> hiddenSysProps = new HashSet<>();
private PluginInfo counterSupplier;
private PluginInfo meterSupplier;
private PluginInfo timerSupplier;
Expand All @@ -170,14 +156,6 @@ public MetricsConfigBuilder setCacheConfig(CacheConfig cacheConfig) {
return this;
}

public MetricsConfigBuilder setHiddenSysProps(Set<String> hiddenSysProps) {
if (hiddenSysProps != null && !hiddenSysProps.isEmpty()) {
this.hiddenSysProps.clear();
this.hiddenSysProps.addAll(hiddenSysProps);
}
return this;
}

public MetricsConfigBuilder setMetricReporterPlugins(PluginInfo[] metricReporterPlugins) {
this.metricReporterPlugins =
metricReporterPlugins != null ? metricReporterPlugins : new PluginInfo[0];
Expand Down Expand Up @@ -228,7 +206,6 @@ public MetricsConfig build() {
return new MetricsConfig(
enabled,
metricReporterPlugins,
hiddenSysProps,
counterSupplier,
meterSupplier,
timerSupplier,
Expand Down
96 changes: 83 additions & 13 deletions solr/core/src/java/org/apache/solr/core/NodeConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,15 @@
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.lucene.search.IndexSearcher;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
Expand Down Expand Up @@ -68,6 +70,9 @@ public class NodeConfig {

private final String modules;

private final Set<String> hiddenSysProps;
private final Predicate<String> hiddenSysPropPattern;

private final PluginInfo shardHandlerFactoryConfig;

private final UpdateShardHandlerConfig updateShardHandlerConfig;
Expand Down Expand Up @@ -145,7 +150,8 @@ private NodeConfig(
Set<Path> allowPaths,
List<String> allowUrls,
String configSetServiceClass,
String modules) {
String modules,
Set<String> hiddenSysProps) {
// all Path params here are absolute and normalized.
this.nodeName = nodeName;
this.coreRootDirectory = coreRootDirectory;
Expand Down Expand Up @@ -180,6 +186,10 @@ private NodeConfig(
this.allowUrls = allowUrls;
this.configSetServiceClass = configSetServiceClass;
this.modules = modules;
this.hiddenSysProps = hiddenSysProps;
this.hiddenSysPropPattern =
Pattern.compile("^(" + String.join("|", hiddenSysProps) + ")$", Pattern.CASE_INSENSITIVE)
.asMatchPredicate();

if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(
Expand Down Expand Up @@ -464,6 +474,25 @@ public String getModules() {
return modules;
}

/** Returns the list of hidden system properties. The list values are regex expressions */
public Set<String> getHiddenSysProps() {
return hiddenSysProps;
}

/** Returns whether a given system property is hidden */
public boolean isSysPropHidden(String sysPropName) {
return hiddenSysPropPattern.test(sysPropName);
}

public static final String REDACTED_SYS_PROP_VALUE = "--REDACTED--";

/** Returns the a system property value, or "--REDACTED--" if the system property is hidden */
public String getRedactedSysPropValue(String sysPropName) {
return hiddenSysPropPattern.test(sysPropName)
? REDACTED_SYS_PROP_VALUE
: System.getProperty(sysPropName);
}

// Finds every jar in each folder and adds it to shardLib, then reloads Lucene SPI
private void addFoldersToSharedLib(Set<String> libDirs) {
boolean modified = false;
Expand Down Expand Up @@ -551,6 +580,7 @@ public static class NodeConfigBuilder {
private Path configSetBaseDirectory;
private String sharedLibDirectory;
private String modules;
private String hiddenSysProps;
private PluginInfo shardHandlerFactoryConfig;
private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT;
private String configSetServiceClass;
Expand Down Expand Up @@ -595,16 +625,17 @@ public static class NodeConfigBuilder {
"org.apache.solr.handler.admin.ConfigSetsHandler";

public static final Set<String> DEFAULT_HIDDEN_SYS_PROPS =
new HashSet<>(
Arrays.asList(
"javax.net.ssl.keyStorePassword",
"javax.net.ssl.trustStorePassword",
"basicauth",
"zkDigestPassword",
"zkDigestReadonlyPassword",
"aws.secretKey", // AWS SDK v1
"aws.secretAccessKey", // AWS SDK v2
"http.proxyPassword"));
Set.of(
"javax\\.net\\.ssl\\.keyStorePassword",
"javax\\.net\\.ssl\\.trustStorePassword",
"basicauth",
"zkDigestPassword",
"zkDigestReadonlyPassword",
"aws\\.secretKey", // AWS SDK v1
"aws\\.secretAccessKey", // AWS SDK v2
"http\\.proxyPassword",
".*password.*",
".*secret.*");

public NodeConfigBuilder(String nodeName, Path solrHome) {
this.nodeName = nodeName;
Expand Down Expand Up @@ -779,6 +810,44 @@ public NodeConfigBuilder setModules(String moduleNames) {
return this;
}

public NodeConfigBuilder setHiddenSysProps(String hiddenSysProps) {
this.hiddenSysProps = hiddenSysProps;
return this;
}

/**
* Finds list of hiddenSysProps requested by system property or environment variable or the
* default
*
* @return set of raw hidden sysProps, may be regex
*/
private Set<String> resolveHiddenSysPropsFromSysPropOrEnvOrDefault(String hiddenSysProps) {
// Fall back to sysprop and env.var if nothing configured through solr.xml
if (!StrUtils.isNotNullOrEmpty(hiddenSysProps)) {
String fromProps = System.getProperty("solr.hiddenSysProps");
// Back-compat for solr 9x
// DEPRECATED: Remove in 10.0
if (StrUtils.isNotNullOrEmpty(fromProps)) {
fromProps = System.getProperty("solr.redaction.system.pattern");
}
String fromEnv = System.getenv("SOLR_HIDDEN_SYS_PROPS");
if (StrUtils.isNotNullOrEmpty(fromProps)) {
hiddenSysProps = fromProps;
} else if (StrUtils.isNotNullOrEmpty(fromEnv)) {
hiddenSysProps = fromEnv;
}
}
Set<String> hiddenSysPropSet = Collections.emptySet();
if (hiddenSysProps != null) {
hiddenSysPropSet =
StrUtils.splitSmart(hiddenSysProps, ',').stream()
.map(String::trim)
.filter(s -> !s.isEmpty())
.collect(Collectors.toSet());
}
return hiddenSysPropSet.isEmpty() ? DEFAULT_HIDDEN_SYS_PROPS : hiddenSysPropSet;
}

public NodeConfig build() {
// if some things weren't set then set them now. Simple primitives are set on the field
// declaration
Expand Down Expand Up @@ -818,7 +887,8 @@ public NodeConfig build() {
allowPaths,
allowUrls,
configSetServiceClass,
modules);
modules,
resolveHiddenSysPropsFromSysPropOrEnvOrDefault(hiddenSysProps));
}

public NodeConfigBuilder setSolrResourceLoader(SolrResourceLoader resourceLoader) {
Expand Down
40 changes: 27 additions & 13 deletions solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ public static NodeConfig fromConfig(
if (cloudConfig != null) configBuilder.setCloudConfig(cloudConfig);
configBuilder.setBackupRepositoryPlugins(
getBackupRepositoryPluginInfos(root.get("backup").getAll("repository")));
// <metrics><hiddenSysProps></metrics> will be removed in Solr 10, but until then, use it if a
// <hiddenSysProps> is not provided under <solr>.
// Remove this line in 10.0
configBuilder.setHiddenSysProps(getHiddenSysProps(root.get("metrics")));
configBuilder.setMetricsConfig(getMetricsConfig(root.get("metrics")));
configBuilder.setFromZookeeper(fromZookeeper);
configBuilder.setDefaultZkHost(defaultZkHost);
Expand Down Expand Up @@ -360,6 +364,9 @@ private static NodeConfig fillSolrSection(NodeConfig.NodeConfigBuilder builder,
case "modules":
builder.setModules(it.txt());
break;
case "hiddenSysProps":
builder.setHiddenSysProps(it.txt());
break;
case "allowPaths":
builder.setAllowPaths(separatePaths(it.txt()));
break;
Expand Down Expand Up @@ -404,6 +411,13 @@ private static List<String> separateStrings(String commaSeparatedString) {
return Arrays.asList(COMMA_SEPARATED_PATTERN.split(commaSeparatedString));
}

private static Set<String> separateStringsToSet(String commaSeparatedString) {
if (StrUtils.isNullOrEmpty(commaSeparatedString)) {
return Collections.emptySet();
}
return Set.of(COMMA_SEPARATED_PATTERN.split(commaSeparatedString));
}

private static Set<Path> separatePaths(String commaSeparatedString) {
if (StrUtils.isNullOrEmpty(commaSeparatedString)) {
return Collections.emptySet();
Expand Down Expand Up @@ -673,11 +687,7 @@ private static MetricsConfig getMetricsConfig(ConfigNode metrics) {
}

PluginInfo[] reporterPlugins = getMetricReporterPluginInfos(metrics);
Set<String> hiddenSysProps = getHiddenSysProps(metrics);
return builder
.setMetricReporterPlugins(reporterPlugins)
.setHiddenSysProps(hiddenSysProps)
.build();
return builder.setMetricReporterPlugins(reporterPlugins).build();
}

private static Object decodeNullValue(Object o) {
Expand Down Expand Up @@ -721,20 +731,24 @@ private static PluginInfo[] getMetricReporterPluginInfos(ConfigNode metrics) {
return configs.toArray(new PluginInfo[configs.size()]);
}

private static Set<String> getHiddenSysProps(ConfigNode metrics) {
/**
* Deprecated as of 9.3, will be removed in 10.0
*
* @param metrics configNode for the metrics
* @return a comma-separated list of hidden Sys Props
*/
@Deprecated(forRemoval = true, since = "9.3")
private static String getHiddenSysProps(ConfigNode metrics) {
ConfigNode p = metrics.get("hiddenSysProps");
if (!p.exists()) return NodeConfig.NodeConfigBuilder.DEFAULT_HIDDEN_SYS_PROPS;
if (!p.exists()) return null;
Set<String> props = new HashSet<>();
p.forEachChild(
it -> {
if (it.name().equals("str") && StrUtils.isNotNullOrEmpty(it.txt())) props.add(it.txt());
if (it.name().equals("str") && StrUtils.isNotNullOrEmpty(it.txt()))
props.add(Pattern.quote(it.txt()));
return Boolean.TRUE;
});
if (props.isEmpty()) {
return NodeConfig.NodeConfigBuilder.DEFAULT_HIDDEN_SYS_PROPS;
} else {
return props;
}
return String.join(",", props);
}

private static PluginInfo getPluginInfo(ConfigNode cfg) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class InfoHandler extends RequestHandlerBase {
public InfoHandler(final CoreContainer coreContainer) {
this.coreContainer = coreContainer;
handlers.put("threads", new ThreadDumpHandler());
handlers.put("properties", new PropertiesRequestHandler());
handlers.put("properties", new PropertiesRequestHandler(coreContainer));
handlers.put("logging", new LoggingHandler(coreContainer));
handlers.put("system", new SystemInfoHandler(coreContainer));
if (coreContainer.getHealthCheckHandler() == null) {
Expand Down
Loading

0 comments on commit 98c1988

Please sign in to comment.