From f63dbd4f31d111fc247e69f91bdef78cee148b22 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 29 Apr 2021 14:45:28 +1000 Subject: [PATCH 1/3] merging from branch_8x --- .../org/apache/solr/core/ConfigOverlay.java | 25 +- .../apache/solr/core/ConfigSetService.java | 7 + .../java/org/apache/solr/core/PluginInfo.java | 24 + .../java/org/apache/solr/core/SolrConfig.java | 564 +++++++++++------- .../org/apache/solr/core/XmlConfigFile.java | 15 +- .../solr/handler/DumpRequestHandler.java | 1 + .../solr/schema/FieldTypePluginLoader.java | 6 +- .../org/apache/solr/schema/IndexSchema.java | 18 +- .../solr/schema/IndexSchemaFactory.java | 4 +- .../org/apache/solr/search/CacheConfig.java | 42 +- .../apache/solr/update/SolrIndexConfig.java | 82 ++- .../org/apache/solr/update/VersionInfo.java | 4 +- .../org/apache/solr/util/DOMConfigNode.java | 10 +- .../org/apache/solr/util/DataConfigNode.java | 71 ++- .../EditableSolrConfigAttributes.json | 4 - .../org/apache/solr/core/TestBadConfig.java | 5 + .../apache/solr/core/TestCodecSupport.java | 6 +- .../apache/solr/core/TestConfLoadPerf.java | 93 +++ .../test/org/apache/solr/core/TestConfig.java | 25 +- .../apache/solr/core/TestConfigOverlay.java | 3 - .../apache/solr/core/TestSimpleTextCodec.java | 2 +- .../component/SuggestComponentTest.java | 122 ++-- .../solr/update/SolrIndexConfigTest.java | 12 +- .../apache/solr/cluster/api/SimpleMap.java | 11 + .../org/apache/solr/common/ConfigNode.java | 133 ++++- .../org/apache/solr/common/util/DOMUtil.java | 8 +- .../solr/common/util/LinkedSimpleHashMap.java | 6 + .../solr/common/util/WrappedSimpleMap.java | 11 + 28 files changed, 910 insertions(+), 404 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java diff --git a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java index 75793a284f85..5ddbe5ed7a65 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java @@ -16,12 +16,12 @@ */ package org.apache.solr.core; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; @@ -62,6 +62,12 @@ public Object getXPathProperty(String xpath, boolean onlyPrimitive) { return Utils.getObjectByPath(props, onlyPrimitive, hierarchy); } + public Object getXPathProperty(List path) { + List hierarchy = new ArrayList<>(); + if(isEditable(true, hierarchy, path) == null) return null; + return Utils.getObjectByPath(props, true, hierarchy); + } + @SuppressWarnings({"unchecked"}) public ConfigOverlay setUserProperty(String key, Object val) { @SuppressWarnings({"rawtypes"}) @@ -180,11 +186,20 @@ public static boolean isEditableProp(String path, boolean isXpath, List @SuppressWarnings({"rawtypes"}) public static Class checkEditable(String path, boolean isXpath, List hierarchy) { - List parts = StrUtils.splitSmart(path, isXpath ? '/' : '.'); + return isEditable(isXpath, hierarchy, StrUtils.splitSmart(path, isXpath ? '/' : '.')); + } + + @SuppressWarnings("rawtypes") + private static Class isEditable(boolean isXpath, List hierarchy, List parts) { Object obj = editable_prop_map; for (int i = 0; i < parts.size(); i++) { String part = parts.get(i); - boolean isAttr = isXpath && part.startsWith("@"); + boolean isAttr = false; + try { + isAttr = isXpath && part.startsWith("@"); + } catch (RuntimeException e) { + throw e; + } if (isAttr) { part = part.substring(1); } @@ -247,6 +262,10 @@ public Map getNamedPlugins(String typ) { return Collections.unmodifiableMap(reqHandlers); } + boolean hasKey(String key) { + return props.containsKey(key); + } + @SuppressWarnings({"unchecked", "rawtypes"}) public ConfigOverlay addNamedPlugin(Map info, String typ) { diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index f5d5c60ca05f..0e6548d2c3ff 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -33,6 +33,7 @@ import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.ConfigNode; import org.apache.solr.common.SolrException; import org.apache.solr.common.StringUtils; import org.apache.solr.common.util.NamedList; @@ -435,4 +436,10 @@ protected NamedList loadConfigSetFlags(CoreDescriptor cd, SolrResourceLoader loa */ public abstract List getAllConfigFiles(String configName) throws IOException; + public interface ConfigResource { + + ConfigNode get() throws Exception; + + } + } diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index 9dff7d265691..b32bac7ca7c5 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.solr.common.ConfigNode; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.util.DOMUtil; import org.apache.solr.common.util.NamedList; @@ -99,6 +100,18 @@ public String toString() { } + public PluginInfo(ConfigNode node, String err,boolean requireName, boolean requireClass) { + type = node.name(); + name = node.requiredStrAttr(NAME,requireName? () -> new RuntimeException(err + ": missing mandatory attribute 'name'"):null); + cName = parseClassName(node.requiredStrAttr(CLASS_NAME, requireClass? () -> new RuntimeException(err + ": missing mandatory attribute 'class'"):null )); + className = cName.className; + pkgName = cName.pkg; + initArgs = DOMUtil.childNodesToNamedList(node); + attributes = node.attributes().asMap(); + children = loadSubPlugins(node); + isFromSolrConfig = true; + + } public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) { type = node.getNodeName(); name = DOMUtil.getAttr(node, NAME, requireName ? err : null); @@ -143,6 +156,17 @@ public PluginInfo(String type, Map map) { isFromSolrConfig = true; } + private List loadSubPlugins(ConfigNode node) { + List children = new ArrayList<>(); + //if there is another sub tag with a non namedlist tag that has to be another plugin + node.forEachChild(nd -> { + if (NL_TAGS.contains(nd.name())) return null; + PluginInfo pluginInfo = new PluginInfo(nd, null, false, false); + if (pluginInfo.isEnabled()) children.add(pluginInfo); + return null; + }); + return children.isEmpty() ? Collections.emptyList() : unmodifiableList(children); + } private List loadSubPlugins(Node node) { List children = new ArrayList<>(); //if there is another sub tag with a non namedlist tag that has to be another plugin diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 804ceab69732..9fafda4edbe5 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -17,8 +17,6 @@ package org.apache.solr.core; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPathConstants; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -36,25 +34,31 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.collect.ImmutableList; import org.apache.commons.io.FileUtils; import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.Version; import org.apache.solr.client.solrj.io.stream.expr.Expressible; import org.apache.solr.cloud.RecoveryStrategy; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.cluster.api.SimpleMap; +import org.apache.solr.common.ConfigNode; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.DOMUtil; import org.apache.solr.common.util.IOUtils; import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.pkg.PackageListeners; @@ -77,12 +81,11 @@ import org.apache.solr.update.UpdateLog; import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; +import org.apache.solr.util.DOMConfigNode; +import org.apache.solr.util.DataConfigNode; import org.apache.solr.util.circuitbreaker.CircuitBreakerManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.PATH; @@ -95,6 +98,7 @@ import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_CLASS; import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME; import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME_IN_OVERLAY; +import static org.apache.solr.core.XmlConfigFile.assertWarnOrFail; /** @@ -102,12 +106,17 @@ * configuration data for a Solr instance -- typically found in * "solrconfig.xml". */ -public class SolrConfig extends XmlConfigFile implements MapSerializable { +public class SolrConfig implements MapSerializable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; + private final String resourceName; + private int znodeVersion; + ConfigNode root; + private final SolrResourceLoader resourceLoader; + private Properties substituteProperties; private RequestParams requestParams; @@ -141,7 +150,7 @@ public enum PluginOpts { * @param name the configuration name used by the loader if the stream is null */ public SolrConfig(Path instanceDir, String name) - throws ParserConfigurationException, IOException, SAXException { + throws IOException { this(new SolrResourceLoader(instanceDir), name, true, null); } @@ -158,6 +167,27 @@ public static SolrConfig readFromResourceLoader(SolrResourceLoader loader, Strin throw new SolrException(ErrorCode.SERVER_ERROR, "Error loading solr config from " + resource, e); } } + private class ResourceProvider implements Function { + int zkVersion; + int hash = -1; + InputStream in; + String fileName; + + ResourceProvider(InputStream in) { + this.in = in; + if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) { + ZkSolrResourceLoader.ZkByteArrayInputStream zkin = (ZkSolrResourceLoader.ZkByteArrayInputStream) in; + zkVersion = zkin.getStat().getVersion(); + hash = Objects.hash(zkVersion, overlay.getZnodeVersion()); + this.fileName = zkin.fileName; + } + } + + @Override + public InputStream apply(String s) { + return in; + } + } /** * Creates a configuration instance from a resource loader, a configuration name and a stream. @@ -168,140 +198,180 @@ public static SolrConfig readFromResourceLoader(SolrResourceLoader loader, Strin * @param isConfigsetTrusted false if configset was uploaded using unsecured configset upload API, true otherwise * @param substitutableProperties optional properties to substitute into the XML */ + @SuppressWarnings("unchecked") private SolrConfig(SolrResourceLoader loader, String name, boolean isConfigsetTrusted, Properties substitutableProperties) - throws ParserConfigurationException, IOException, SAXException { - // insist we have non-null substituteProperties; it might get overlayed - super(loader, name, null, "/config/", substitutableProperties == null ? new Properties() : substitutableProperties); + throws IOException { + this.resourceLoader = loader; + this.resourceName = name; + this.substituteProperties = substitutableProperties; getOverlay();//just in case it is not initialized - getRequestParams(); - initLibs(loader, isConfigsetTrusted); - luceneMatchVersion = SolrConfig.parseLuceneVersionString(getVal(IndexSchema.LUCENE_MATCH_VERSION_PARAM, true)); - log.info("Using Lucene MatchVersion: {}", luceneMatchVersion); - - String indexConfigPrefix; - - // Old indexDefaults and mainIndex sections are deprecated and fails fast for luceneMatchVersion=>LUCENE_4_0_0. - // For older solrconfig.xml's we allow the old sections, but never mixed with the new - boolean hasDeprecatedIndexConfig = (getNode("indexDefaults", false) != null) || (getNode("mainIndex", false) != null); - if (hasDeprecatedIndexConfig) { - throw new SolrException(ErrorCode.FORBIDDEN, " and configuration sections are discontinued. Use instead."); - } else { - indexConfigPrefix = "indexConfig"; - } - assertWarnOrFail("The config has been discontinued and NRT mode is always used by Solr." + - " This config will be removed in future versions.", getNode(indexConfigPrefix + "/nrtMode", false) == null, - true - ); - assertWarnOrFail("Solr no longer supports forceful unlocking via the 'unlockOnStartup' option. "+ - "This is no longer necessary for the default lockType except in situations where "+ - "it would be dangerous and should not be done. For other lockTypes and/or "+ - "directoryFactory options it may also be dangerous and users must resolve "+ - "problematic locks manually.", - null == getNode(indexConfigPrefix + "/unlockOnStartup", false), - true // 'fail' in trunk - ); - - // Parse indexConfig section, using mainIndex as backup in case old config is used - indexConfig = new SolrIndexConfig(this, "indexConfig", null); - - booleanQueryMaxClauseCount = getInt("query/maxBooleanClauses", IndexSearcher.getMaxClauseCount()); - if (IndexSearcher.getMaxClauseCount() < booleanQueryMaxClauseCount) { - log.warn("solrconfig.xml: of {} is greater than global limit of {} {}" - , booleanQueryMaxClauseCount, IndexSearcher.getMaxClauseCount() - , "and will have no effect set 'maxBooleanClauses' in solr.xml to increase global limit"); - } - - // Warn about deprecated / discontinued parameters - // boolToFilterOptimizer has had no effect since 3.1 - if (get("query/boolTofilterOptimizer", null) != null) - log.warn("solrconfig.xml: is currently not implemented and has no effect."); - if (get("query/HashDocSet", null) != null) - log.warn("solrconfig.xml: is deprecated and no longer used."); + // insist we have non-null substituteProperties; it might get overlaid + Map configCache =null; + if (loader.getCoreContainer() != null && loader.getCoreContainer().getObjectCache() != null) { + configCache = (Map) loader.getCoreContainer().getObjectCache() + .computeIfAbsent(ConfigSetService.ConfigResource.class.getName(), s -> new ConcurrentHashMap<>()); + ResourceProvider rp = new ResourceProvider(loader.openResource(name)); + IndexSchemaFactory.VersionedConfig cfg = rp.fileName == null ? null : configCache.get(rp.fileName); + if (cfg != null) { + if (rp.hash != -1) { + if (rp.hash == cfg.version) { + log.debug("LOADED_FROM_CACHE"); + root = cfg.data; + } else { + readXml(loader, name, configCache, rp); + } + } + } + } + if(root == null) { + readXml(loader, name, configCache,new ResourceProvider(loader.openResource(name)) ); + } + ConfigNode.SUBSTITUTES.set(key -> { + if (substitutableProperties != null && substitutableProperties.containsKey(key)) { + return substitutableProperties.getProperty(key); + } else { + Object o = overlay.getUserProps().get(key); + return o == null ? null : o.toString(); + } + }); + try { + getRequestParams(); + initLibs(loader, isConfigsetTrusted); + String val = root.child(IndexSchema.LUCENE_MATCH_VERSION_PARAM, + () -> new RuntimeException("Missing: " + IndexSchema.LUCENE_MATCH_VERSION_PARAM)).txt(); + + luceneMatchVersion = SolrConfig.parseLuceneVersionString(val); + log.info("Using Lucene MatchVersion: {}", luceneMatchVersion); + + String indexConfigPrefix; + + // Old indexDefaults and mainIndex sections are deprecated and fails fast for luceneMatchVersion=>LUCENE_4_0_0. + // For older solrconfig.xml's we allow the old sections, but never mixed with the new + boolean hasDeprecatedIndexConfig = get("indexDefaults").exists() || get("mainIndex").exists(); + if (hasDeprecatedIndexConfig) { + throw new SolrException(ErrorCode.FORBIDDEN, " and configuration sections are discontinued. Use instead."); + } else { + indexConfigPrefix = "indexConfig"; + } + assertWarnOrFail("The config has been discontinued and NRT mode is always used by Solr." + + " This config will be removed in future versions.", get(indexConfigPrefix).get("nrtMode").isNull(), + true + ); + assertWarnOrFail("Solr no longer supports forceful unlocking via the 'unlockOnStartup' option. " + + "This is no longer necessary for the default lockType except in situations where " + + "it would be dangerous and should not be done. For other lockTypes and/or " + + "directoryFactory options it may also be dangerous and users must resolve " + + "problematic locks manually.", + !get(indexConfigPrefix).get("unlockOnStartup").exists(), + true // 'fail' in trunk + ); + + // Parse indexConfig section, using mainIndex as backup in case old config is used + indexConfig = new SolrIndexConfig(get("indexConfig"), null); + + booleanQueryMaxClauseCount = get("query").get("maxBooleanClauses").intVal(BooleanQuery.getMaxClauseCount()); + if (IndexSearcher.getMaxClauseCount() < booleanQueryMaxClauseCount) { + log.warn("solrconfig.xml: of {} is greater than global limit of {} and will have no effect {}" + , booleanQueryMaxClauseCount, BooleanQuery.getMaxClauseCount() + , "set 'maxBooleanClauses' in solr.xml to increase global limit"); + } + + // Warn about deprecated / discontinued parameters + // boolToFilterOptimizer has had no effect since 3.1 + if (get("query").get("boolTofilterOptimizer").exists()) + log.warn("solrconfig.xml: is currently not implemented and has no effect."); + if (get("query").get("HashDocSet").exists()) + log.warn("solrconfig.xml: is deprecated and no longer used."); // TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search() // filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false); // filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32); // filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f); - useFilterForSortedQuery = getBool("query/useFilterForSortedQuery", false); - queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1)); - queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE); - enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false); - - filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache"); - queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache"); - documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache"); - CacheConfig conf = CacheConfig.getConfig(this, "query/fieldValueCache"); - if (conf == null) { - Map args = new HashMap<>(); - args.put(NAME, "fieldValueCache"); - args.put("size", "10000"); - args.put("initialSize", "10"); - args.put("showItems", "-1"); - conf = new CacheConfig(CaffeineCache.class, args, null); - } - fieldValueCacheConfig = conf; - useColdSearcher = getBool("query/useColdSearcher", false); - dataDir = get("dataDir", null); - if (dataDir != null && dataDir.length() == 0) dataDir = null; - - - org.apache.solr.search.SolrIndexSearcher.initRegenerators(this); - - if (get("jmx", null) != null) { - log.warn("solrconfig.xml: is no longer supported, use solr.xml:/metrics/reporter section instead"); - } - - httpCachingConfig = new HttpCachingConfig(this); - - maxWarmingSearchers = getInt("query/maxWarmingSearchers", 1); - slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1); - for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin); - - Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); - List caches = getPluginInfos(SolrCache.class.getName()); - if (!caches.isEmpty()) { - for (PluginInfo c : caches) { - userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null)); + useFilterForSortedQuery = get("query").get("useFilterForSortedQuery").boolVal(false); + queryResultWindowSize = Math.max(1, get("query").get("queryResultWindowSize").intVal(1)); + queryResultMaxDocsCached = get("query").get("queryResultMaxDocsCached").intVal(Integer.MAX_VALUE); + enableLazyFieldLoading = get("query").get("enableLazyFieldLoading").boolVal(false); + + filterCacheConfig = CacheConfig.getConfig(this, get("query").get("filterCache"), "query/filterCache"); + queryResultCacheConfig = CacheConfig.getConfig(this, get("query").get("queryResultCache"), "query/queryResultCache"); + documentCacheConfig = CacheConfig.getConfig(this, get("query").get("documentCache"), "query/documentCache"); + CacheConfig conf = CacheConfig.getConfig(this, get("query").get("fieldValueCache"), "query/fieldValueCache"); + if (conf == null) { + Map args = new HashMap<>(); + args.put(NAME, "fieldValueCache"); + args.put("size", "10000"); + args.put("initialSize", "10"); + args.put("showItems", "-1"); + conf = new CacheConfig(CaffeineCache.class, args, null); } - } - this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); + fieldValueCacheConfig = conf; + useColdSearcher = get("query").get("useColdSearcher").boolVal(false); + dataDir = get("dataDir").txt(); + if (dataDir != null && dataDir.length() == 0) dataDir = null; + - updateHandlerInfo = loadUpdatehandlerInfo(); + org.apache.solr.search.SolrIndexSearcher.initRegenerators(this); + + if (get("jmx").exists()) { + log.warn("solrconfig.xml: is no longer supported, use solr.xml:/metrics/reporter section instead"); + } - multipartUploadLimitKB = getInt( - "requestDispatcher/requestParsers/@multipartUploadLimitInKB", Integer.MAX_VALUE); - if (multipartUploadLimitKB == -1) multipartUploadLimitKB = Integer.MAX_VALUE; + httpCachingConfig = new HttpCachingConfig(this); - formUploadLimitKB = getInt( - "requestDispatcher/requestParsers/@formdataUploadLimitInKB", Integer.MAX_VALUE); - if (formUploadLimitKB == -1) formUploadLimitKB = Integer.MAX_VALUE; + maxWarmingSearchers = get("query").get("maxWarmingSearchers").intVal(1); + slowQueryThresholdMillis = get("query").get("slowQueryThresholdMillis").intVal(-1); + for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin); + + Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache", + get("query").getAll("cache")); + List caches = getPluginInfos(SolrCache.class.getName()); + if (!caches.isEmpty()) { + for (PluginInfo c : caches) { + userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null)); + } + } + this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); - enableRemoteStreams = getBool( - "requestDispatcher/requestParsers/@enableRemoteStreaming", false); + updateHandlerInfo = loadUpdatehandlerInfo(); - enableStreamBody = getBool( - "requestDispatcher/requestParsers/@enableStreamBody", false); + multipartUploadLimitKB = get("requestDispatcher").get("requestParsers").intAttr("multipartUploadLimitInKB", Integer.MAX_VALUE); + if (multipartUploadLimitKB == -1) multipartUploadLimitKB = Integer.MAX_VALUE; - handleSelect = getBool( - "requestDispatcher/@handleSelect", false); + formUploadLimitKB = get("requestDispatcher").get("requestParsers").intAttr("formdataUploadLimitInKB", Integer.MAX_VALUE); + if (formUploadLimitKB == -1) formUploadLimitKB = Integer.MAX_VALUE; - addHttpRequestToContext = getBool( - "requestDispatcher/requestParsers/@addHttpRequestToContext", false); + enableRemoteStreams = get("requestDispatcher").get("requestParsers").boolAttr("enableRemoteStreaming", false); - List argsInfos = getPluginInfos(InitParams.class.getName()); - if (argsInfos != null) { - Map argsMap = new HashMap<>(); - for (PluginInfo p : argsInfos) { - InitParams args = new InitParams(p); - argsMap.put(args.name == null ? String.valueOf(args.hashCode()) : args.name, args); + enableStreamBody = get("requestDispatcher").get("requestParsers").boolAttr("enableStreamBody", false); + + handleSelect = get("requestDispatcher").boolAttr("handleSelect", false); + addHttpRequestToContext = get("requestDispatcher").get("requestParsers").boolAttr("addHttpRequestToContext", false); + + List argsInfos = getPluginInfos(InitParams.class.getName()); + if (argsInfos != null) { + Map argsMap = new HashMap<>(); + for (PluginInfo p : argsInfos) { + InitParams args = new InitParams(p); + argsMap.put(args.name == null ? String.valueOf(args.hashCode()) : args.name, args); + } + this.initParams = Collections.unmodifiableMap(argsMap); } - this.initParams = Collections.unmodifiableMap(argsMap); + solrRequestParsers = new SolrRequestParsers(this); + log.debug("Loaded SolrConfig: {}", name); + } finally { + ConfigNode.SUBSTITUTES.remove(); } + } - solrRequestParsers = new SolrRequestParsers(this); - log.debug("Loaded SolrConfig: {}", name); + private void readXml(SolrResourceLoader loader, String name, Map configCache, ResourceProvider rp) throws IOException { + XmlConfigFile xml = new XmlConfigFile(loader,rp, name, null, "/config/", null); + root = new DataConfigNode(new DOMConfigNode(xml.getDocument().getDocumentElement())); + this.znodeVersion = rp.zkVersion; + if(configCache != null && rp.fileName !=null) { + configCache.put(rp.fileName, new IndexSchemaFactory.VersionedConfig(rp.hash, root)); + } } private static final AtomicBoolean versionWarningAlreadyLogged = new AtomicBoolean(false); @@ -340,20 +410,25 @@ public static final Version parseLuceneVersionString(final String matchVersion) // and even then -- only if there is a single SpellCheckComponent // because of queryConverter.setIndexAnalyzer .add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS)) - // this is hackish, since it picks up all SolrEventListeners, - // regardless of when/how/why they are used (or even if they are - // declared outside of the appropriate context) but there's no nice - // way around that in the PluginInfo framework + // this is hackish, since it picks up all SolrEventListeners, + // regardless of when/how/why they are used (or even if they are + // declared outside of the appropriate context) but there's no nice + // way around that in the PluginInfo framework .add(new SolrPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK, REQUIRE_NAME_IN_OVERLAY)) - .add(new SolrPluginInfo(SolrEventListener.class, "//listener", REQUIRE_CLASS, MULTI_OK, REQUIRE_NAME_IN_OVERLAY)) + .add(new SolrPluginInfo(it -> { + List result = new ArrayList<>(); + result.addAll(it.get("query").getAll("listener")); + result.addAll( it.get("updateHandler").getAll("listener")); + return result; + }, SolrEventListener.class, "//listener", REQUIRE_CLASS, MULTI_OK, REQUIRE_NAME_IN_OVERLAY)) .add(new SolrPluginInfo(DirectoryFactory.class, "directoryFactory", REQUIRE_CLASS)) .add(new SolrPluginInfo(RecoveryStrategy.Builder.class, "recoveryStrategy")) - .add(new SolrPluginInfo(IndexDeletionPolicy.class, "indexConfig/deletionPolicy", REQUIRE_CLASS)) + .add(new SolrPluginInfo(it -> it.get("indexConfig").getAll("deletionPolicy"), IndexDeletionPolicy.class, "indexConfig/deletionPolicy", REQUIRE_CLASS)) .add(new SolrPluginInfo(CodecFactory.class, "codecFactory", REQUIRE_CLASS)) .add(new SolrPluginInfo(IndexReaderFactory.class, "indexReaderFactory", REQUIRE_CLASS)) .add(new SolrPluginInfo(UpdateRequestProcessorChain.class, "updateRequestProcessorChain", MULTI_OK)) - .add(new SolrPluginInfo(UpdateLog.class, "updateHandler/updateLog")) + .add(new SolrPluginInfo(it -> it.get("updateHandler").getAll("updateLog"), UpdateLog.class, "updateHandler/updateLog")) .add(new SolrPluginInfo(IndexSchemaFactory.class, "schemaFactory", REQUIRE_CLASS)) .add(new SolrPluginInfo(RestManager.class, "restManager")) .add(new SolrPluginInfo(StatsCache.class, "statsCache", REQUIRE_CLASS)) @@ -373,10 +448,18 @@ public static class SolrPluginInfo { public final Class clazz; public final String tag; public final Set options; + final Function> configReader; @SuppressWarnings({"unchecked", "rawtypes"}) private SolrPluginInfo(Class clz, String tag, PluginOpts... opts) { + this(solrConfig -> solrConfig.root.getAll(null, tag), clz, tag, opts); + + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private SolrPluginInfo(Function> configReader, Class clz, String tag, PluginOpts... opts) { + this.configReader = configReader; this.clazz = clz; this.tag = tag; this.options = opts == null ? Collections.EMPTY_SET : EnumSet.of(NOOP, opts); @@ -405,7 +488,7 @@ public static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) { // hopefully no problem, assume no overlay.json file return new ConfigOverlay(Collections.EMPTY_MAP, -1); } - + int version = 0; // will be always 0 for file based resourceLoader if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) { version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion(); @@ -428,15 +511,15 @@ public Map getInitParams() { } protected UpdateHandlerInfo loadUpdatehandlerInfo() { - return new UpdateHandlerInfo(get("updateHandler/@class", null), - getInt("updateHandler/autoCommit/maxDocs", -1), - getInt("updateHandler/autoCommit/maxTime", -1), - convertHeapOptionStyleConfigStringToBytes(get("updateHandler/autoCommit/maxSize", "")), - getBool("updateHandler/indexWriter/closeWaitsForMerges", true), - getBool("updateHandler/autoCommit/openSearcher", true), - getInt("updateHandler/autoSoftCommit/maxDocs", -1), - getInt("updateHandler/autoSoftCommit/maxTime", -1), - getBool("updateHandler/commitWithin/softCommit", true)); + return new UpdateHandlerInfo( get("updateHandler").attr("class"), + get("updateHandler").get("autoCommit").get("maxDocs").intVal( -1), + get("updateHandler").get("autoCommit").get("maxTime").intVal( -1), + convertHeapOptionStyleConfigStringToBytes(get("updateHandler").get("autoCommit").get("maxSize").txt()), + get("updateHandler").get("indexWriter").get("closeWaitsForMerges").boolVal(true), + get("updateHandler").get("autoCommit").get("openSearcher").boolVal(true), + get("updateHandler").get("autoSoftCommit").get("maxDocs").intVal(-1), + get("updateHandler").get("autoSoftCommit").get("maxTime").intVal(-1), + get("updateHandler").get("commitWithin").get("softCommit").boolVal(true)); } /** @@ -447,7 +530,7 @@ protected UpdateHandlerInfo loadUpdatehandlerInfo() { * @return the size, in bytes. -1 if the given config string is empty */ protected static long convertHeapOptionStyleConfigStringToBytes(String configStr) { - if (configStr.isEmpty()) { + if (configStr== null || configStr.isEmpty()) { return -1; } long multiplier = 1; @@ -481,7 +564,7 @@ private void loadPluginInfo(SolrPluginInfo pluginInfo) { boolean requireName = pluginInfo.options.contains(REQUIRE_NAME); boolean requireClass = pluginInfo.options.contains(REQUIRE_CLASS); - List result = readPluginInfos(pluginInfo.tag, requireName, requireClass); + List result = readPluginInfos(pluginInfo, requireName, requireClass); if (1 < result.size() && !pluginInfo.options.contains(MULTI_OK)) { throw new SolrException @@ -492,11 +575,10 @@ private void loadPluginInfo(SolrPluginInfo pluginInfo) { if (!result.isEmpty()) pluginStore.put(pluginInfo.clazz.getName(), result); } - public List readPluginInfos(String tag, boolean requireName, boolean requireClass) { + public List readPluginInfos(SolrPluginInfo info, boolean requireName, boolean requireClass) { ArrayList result = new ArrayList<>(); - NodeList nodes = (NodeList) evaluate(tag, XPathConstants.NODESET); - for (int i = 0; i < nodes.getLength(); i++) { - PluginInfo pluginInfo = new PluginInfo(nodes.item(i), "[solrconfig.xml] " + tag, requireName, requireClass); + for (ConfigNode node : info.configReader.apply(this)) { + PluginInfo pluginInfo = new PluginInfo(node, "[solrconfig.xml] " + info.tag, requireName, requireClass); if (pluginInfo.isEnabled()) result.add(pluginInfo); } return result; @@ -545,12 +627,6 @@ public HttpCachingConfig getHttpCachingConfig() { public static class HttpCachingConfig implements MapSerializable { - /** - * config xpath prefix for getting HTTP Caching options - */ - private final static String CACHE_PRE - = "requestDispatcher/httpCaching/"; - /** * For extracting Expires "ttl" from config */ @@ -565,7 +641,7 @@ public Map toMap(Map map) { "cacheControl", cacheControlHeader); } - public static enum LastModFrom { + public enum LastModFrom { OPENTIME, DIRLASTMOD, BOGUS; /** @@ -586,18 +662,20 @@ public static LastModFrom parse(final String s) { private final String cacheControlHeader; private final Long maxAge; private final LastModFrom lastModFrom; + private ConfigNode configNode; private HttpCachingConfig(SolrConfig conf) { + configNode = conf.root; - never304 = conf.getBool(CACHE_PRE + "@never304", false); + //"requestDispatcher/httpCaching/"; + never304 = get("requestDispatcher").get("httpCaching").boolAttr("never304", false); - etagSeed = conf.get(CACHE_PRE + "@etagSeed", "Solr"); + etagSeed = get("requestDispatcher").get("httpCaching").attr("etagSeed", "Solr"); - lastModFrom = LastModFrom.parse(conf.get(CACHE_PRE + "@lastModFrom", - "openTime")); + lastModFrom = LastModFrom.parse(get("requestDispatcher").get("httpCaching").attr("lastModFrom","openTime")); - cacheControlHeader = conf.get(CACHE_PRE + "cacheControl", null); + cacheControlHeader = get("requestDispatcher").get("httpCaching").get("cacheControl").txt(); Long tmp = null; // maxAge if (null != cacheControlHeader) { @@ -615,6 +693,9 @@ private HttpCachingConfig(SolrConfig conf) { maxAge = tmp; } + private ConfigNode get(String name){ + return configNode.get(name); + } public boolean isNever304() { return never304; @@ -763,23 +844,23 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { } } - NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET); - if (nodes != null && nodes.getLength() > 0) { + List nodes = root.getAll("lib"); + if (nodes != null && nodes.size() > 0) { if (!isConfigsetTrusted) { throw new SolrException(ErrorCode.UNAUTHORIZED, - "The configset for this collection was uploaded without any authentication in place," - + " and use of is not available for collections with untrusted configsets. To use this component, re-upload the configset" - + " after enabling authentication and authorization."); + "The configset for this collection was uploaded without any authentication in place," + + " and use of is not available for collections with untrusted configsets. To use this component, re-upload the configset" + + " after enabling authentication and authorization."); } - for (int i = 0; i < nodes.getLength(); i++) { - Node node = nodes.item(i); - String baseDir = DOMUtil.getAttr(node, "dir"); - String path = DOMUtil.getAttr(node, PATH); + for (int i = 0; i < nodes.size(); i++) { + ConfigNode node = nodes.get(i); + String baseDir = node.attr("dir"); + String path = node.attr(PATH); if (null != baseDir) { // :TODO: add support for a simpler 'glob' mutually exclusive of regex Path dir = instancePath.resolve(baseDir); - String regex = DOMUtil.getAttr(node, "regex"); + String regex = node.attr("regex"); try { if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir)); @@ -831,42 +912,11 @@ public boolean isEnableStreamBody() { return enableStreamBody; } - @Override - public int getInt(String path) { - return getInt(path, 0); - } - - @Override - public int getInt(String path, int def) { - Object val = overlay.getXPathProperty(path); - if (val != null) return Integer.parseInt(val.toString()); - return super.getInt(path, def); - } - - @Override - public boolean getBool(String path, boolean def) { - Object val = overlay.getXPathProperty(path); - if (val != null) return Boolean.parseBoolean(val.toString()); - return super.getBool(path, def); - } - - @Override - public String get(String path) { - Object val = overlay.getXPathProperty(path, true); - return val != null ? val.toString() : super.get(path); - } - - @Override - public String get(String path, String def) { - Object val = overlay.getXPathProperty(path, true); - return val != null ? val.toString() : super.get(path, def); - - } @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Map toMap(Map result) { - if (getZnodeVersion() > -1) result.put(ZNODEVER, getZnodeVersion()); + if (znodeVersion > -1) result.put(ZNODEVER, znodeVersion); if(luceneMatchVersion != null) result.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM, luceneMatchVersion.toString()); result.put("updateHandler", getUpdateHandlerInfo()); Map m = new LinkedHashMap(); @@ -927,11 +977,10 @@ private void addCacheConfig(Map queryMap, CacheConfig... cache) { } - @Override public Properties getSubstituteProperties() { Map p = getOverlay().getUserProps(); - if (p == null || p.isEmpty()) return super.getSubstituteProperties(); - Properties result = new Properties(super.getSubstituteProperties()); + if (p == null || p.isEmpty()) return substituteProperties; + Properties result = new Properties(substituteProperties); result.putAll(p); return result; } @@ -940,7 +989,7 @@ public Properties getSubstituteProperties() { public ConfigOverlay getOverlay() { if (overlay == null) { - overlay = getConfigOverlay(getResourceLoader()); + overlay = getConfigOverlay(resourceLoader); } return overlay; } @@ -969,11 +1018,120 @@ public String maxPackageVersion(String pkg) { } public RequestParams refreshRequestParams() { - requestParams = RequestParams.getFreshRequestParams(getResourceLoader(), requestParams); + requestParams = RequestParams.getFreshRequestParams(resourceLoader, requestParams); if (log.isDebugEnabled()) { log.debug("current version of requestparams : {}", requestParams.getZnodeVersion()); } return requestParams; } + public SolrResourceLoader getResourceLoader() { + return resourceLoader; + } + + public int getZnodeVersion() { + return znodeVersion; + } + + public String getName() { + return resourceName; + } + + public String getResourceName() { + return resourceName; + } + + /**fetches a child node by name. An "empty node" is returned if the child does not exist + * This never returns a null + * + * + */ + public ConfigNode get(String name) { + if (!overlay.hasKey(name)) { + //there is no overlay + return root.get(name); + } + return new OverlaidConfigNode(name, null,root.get(name)); + } + + public ConfigNode get(String name, Predicate test) { + return root.get(name, test); + } + private class OverlaidConfigNode implements ConfigNode { + + private final String _name; + private final ConfigNode delegate; + private final OverlaidConfigNode parent; + + private OverlaidConfigNode(String name, OverlaidConfigNode parent, ConfigNode delegate) { + this._name = name; + this.delegate = delegate; + this.parent = parent; + } + + private List path(List path) { + if(path== null) path = new ArrayList<>(5); + try { + if (parent != null) return parent.path(path); + } finally { + path.add(_name); + } + return path; + } + + @Override + public ConfigNode get(String name) { + return wrap(delegate.get(name), name); + } + + private ConfigNode wrap(ConfigNode n, String name) { + return new OverlaidConfigNode(name,this, n); + } + + @Override + public ConfigNode get(String name, Predicate test) { + return wrap(delegate.get(name, test), name); + } + + @Override + public String txt() { + return overlayText(delegate.txt(), null); + } + + @Override + public ConfigNode get(String name, int idx) { + return wrap(delegate.get(name, idx), name); + } + + @Override + public String name() { + return delegate.name(); + } + @Override + public SimpleMap attributes() { + return delegate.attributes(); + } + + @Override + public boolean exists() { + return delegate.exists(); + } + + @Override + public String attr(String name) { + return overlayText(delegate.attr(name),name); + } + + private String overlayText(String def, String appendToPath) { + List path = path(null); + if(appendToPath !=null) path.add(appendToPath); + Object val = overlay.getXPathProperty(path); + return val ==null? def: val.toString(); + } + + @Override + public void forEachChild(Function fun) { + delegate.forEachChild(fun); + } + } } diff --git a/solr/core/src/java/org/apache/solr/core/XmlConfigFile.java b/solr/core/src/java/org/apache/solr/core/XmlConfigFile.java index 5971eb5bd26d..6a545d47b020 100644 --- a/solr/core/src/java/org/apache/solr/core/XmlConfigFile.java +++ b/solr/core/src/java/org/apache/solr/core/XmlConfigFile.java @@ -36,6 +36,7 @@ import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.solr.cloud.ZkSolrResourceLoader; @@ -85,6 +86,15 @@ public XmlConfigFile(SolrResourceLoader loader, String name, InputSource is, Str { this(loader, name, is, prefix, null); } + public XmlConfigFile(SolrResourceLoader loader, String name, InputSource is, String prefix, Properties substituteProps) throws ParserConfigurationException, IOException, SAXException{ + this(loader, s -> { + try { + return loader.openResource(s); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + },name, is, prefix, substituteProps); + } /** * Builds a config: @@ -103,8 +113,7 @@ public XmlConfigFile(SolrResourceLoader loader, String name, InputSource is, Str * @param prefix an optional prefix that will be prepended to all non-absolute xpath expressions * @param substituteProps optional property substitution */ - public XmlConfigFile(SolrResourceLoader loader, String name, InputSource is, String prefix, Properties substituteProps) throws ParserConfigurationException, IOException, SAXException - { + public XmlConfigFile(SolrResourceLoader loader, Function fileSupplier, String name, InputSource is, String prefix, Properties substituteProps) throws IOException { if (null == loader) throw new NullPointerException("loader"); this.loader = loader; @@ -115,7 +124,7 @@ public XmlConfigFile(SolrResourceLoader loader, String name, InputSource is, Str javax.xml.parsers.DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); if (is == null) { - InputStream in = loader.openResource(name); + InputStream in = fileSupplier.apply(name); if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) { zkVersion = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion(); log.debug("loaded config {} with version {} ",name,zkVersion); diff --git a/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java index 4ce5fa57dab7..c99ca79430f3 100644 --- a/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java @@ -36,6 +36,7 @@ public class DumpRequestHandler extends RequestHandlerBase { + @Override @SuppressWarnings({"unchecked"}) public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java index 345a2cfe23b8..b1c72e20cfa4 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java @@ -189,9 +189,9 @@ private Analyzer readAnalyzer(ConfigNode node) { // check for all of these up front, so we can error if used in // conjunction with an explicit analyzer class. - List charFilterNodes = node.children("charFilter"); - List tokenizerNodes = node.children("tokenizer"); - List tokenFilterNodes = node.children("filter"); + List charFilterNodes = node.getAll("charFilter"); + List tokenizerNodes = node.getAll("tokenizer"); + List tokenFilterNodes = node.getAll("filter"); if (analyzerName != null) { diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 19c1de2237ab..d65159e79a56 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -172,7 +172,9 @@ public IndexSchema(String name, IndexSchemaFactory.ConfigResource schemaResource this(luceneVersion, resourceLoader, substitutableProperties); this.resourceName = Objects.requireNonNull(name); - ConfigNode.SUBSTITUTES.set(substitutableProperties::getProperty); + ConfigNode.SUBSTITUTES.set(key -> substitutableProperties == null ? + null : + substitutableProperties.getProperty(key)); try { readSchema(schemaResource); loader.inform(loader); @@ -513,9 +515,9 @@ protected void readSchema(IndexSchemaFactory.ConfigResource is) { // load the Field Types final FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, fieldTypes, schemaAware); - List fTypes = rootNode.children(null, FIELDTYPE_KEYS); + List fTypes = rootNode.getAll(null, FIELDTYPE_KEYS); ConfigNode types = rootNode.child(TYPES); - if(types != null) fTypes.addAll(types.children(null, FIELDTYPE_KEYS)); + if(types != null) fTypes.addAll(types.getAll(null, FIELDTYPE_KEYS)); typeLoader.load(solrClassLoader, fTypes); // load the fields @@ -560,7 +562,7 @@ protected void readSchema(IndexSchemaFactory.ConfigResource is) { if (node==null) { log.warn("no {} specified in schema.", UNIQUE_KEY); } else { - uniqueKeyField=getIndexedField(node.textValue().trim()); + uniqueKeyField=getIndexedField(node.txt().trim()); uniqueKeyFieldName=uniqueKeyField.getName(); uniqueKeyFieldType=uniqueKeyField.getType(); @@ -647,10 +649,10 @@ protected synchronized Map loadFields(ConfigNode n) { ArrayList dFields = new ArrayList<>(); - List nodes = n.children(null, FIELD_KEYS); + List nodes = n.getAll(null, FIELD_KEYS); ConfigNode child = n.child(FIELDS); if(child != null) { - nodes.addAll(child.children(null, FIELD_KEYS)); + nodes.addAll(child.getAll(null, FIELD_KEYS)); } for (ConfigNode node : nodes) { @@ -730,10 +732,10 @@ protected static DynamicField[] dynamicFieldListToSortedArray(List * Loads the copy fields */ protected synchronized void loadCopyFields(ConfigNode n) { - List nodes = n.children(COPY_FIELD); + List nodes = n.getAll(COPY_FIELD); ConfigNode f = n.child(FIELDS); if (f != null) { - List c = f.children(COPY_FIELD); + List c = f.getAll(COPY_FIELD); if (nodes.isEmpty()) nodes = c; else nodes.addAll(c); } diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java index c9d31871dcab..01319439d895 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java @@ -141,8 +141,8 @@ public interface ConfigResource { } public static class VersionedConfig { - final int version; - final ConfigNode data; + public final int version; + public final ConfigNode data; public VersionedConfig(int version, ConfigNode data) { this.version = version; diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index af1b87f35681..45ec01a2623c 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -16,7 +16,6 @@ */ package org.apache.solr.search; -import javax.xml.xpath.XPathConstants; import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.HashMap; @@ -25,19 +24,15 @@ import java.util.Map; import java.util.function.Supplier; +import org.apache.solr.common.ConfigNode; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.DOMUtil; import org.apache.solr.common.util.StrUtils; - import org.apache.solr.core.PluginInfo; - import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; import static org.apache.solr.common.params.CommonParams.NAME; @@ -70,7 +65,8 @@ public class CacheConfig implements MapSerializable{ private String regenImpl; - public CacheConfig() {} + public CacheConfig() { + } @SuppressWarnings({"rawtypes"}) public CacheConfig(Class clazz, Map args, CacheRegenerator regenerator) { @@ -88,15 +84,13 @@ public void setRegenerator(CacheRegenerator regenerator) { this.regenerator = regenerator; } - public static Map getMultipleConfigs(SolrConfig solrConfig, String configPath) { - NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET); - if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>(); - Map result = new HashMap<>(nodes.getLength()); - for (int i = 0; i < nodes.getLength(); i++) { - Node node = nodes.item(i); - if ("true".equals(DOMUtil.getAttrOrDefault(node, "enabled", "true"))) { - CacheConfig config = getConfig(solrConfig, node.getNodeName(), - DOMUtil.toMap(node.getAttributes()), configPath); + public static Map getMultipleConfigs(SolrConfig solrConfig, String configPath, List nodes) { + if (nodes == null || nodes.size() == 0) return new LinkedHashMap<>(); + Map result = new HashMap<>(nodes.size()); + for (int i = 0; i < nodes.size(); i++) { + ConfigNode node = nodes.get(i); + if (node.boolAttr("enabled", true)) { + CacheConfig config = getConfig(solrConfig, node.name(), node.attributes().asMap(), configPath); result.put(config.args.get(NAME), config); } } @@ -104,20 +98,20 @@ public static Map getMultipleConfigs(SolrConfig solrConfig, } - public static CacheConfig getConfig(SolrConfig solrConfig, String xpath) { - Node node = solrConfig.getNode(xpath, false); - if(node == null || !"true".equals(DOMUtil.getAttrOrDefault(node, "enabled", "true"))) { - Map m = solrConfig.getOverlay().getEditableSubProperties(xpath); - if(m==null) return null; + @SuppressWarnings({"unchecked"}) + public static CacheConfig getConfig(SolrConfig solrConfig, ConfigNode node, String xpath) { + if (!node.exists() || !"true".equals(node.attributes().get("enabled", "true"))) { + Map m = solrConfig.getOverlay().getEditableSubProperties(xpath); + if (m == null) return null; List parts = StrUtils.splitSmart(xpath, '/'); - return getConfig(solrConfig,parts.get(parts.size()-1), Collections.emptyMap(), xpath); + return getConfig(solrConfig, parts.get(parts.size() - 1), Collections.EMPTY_MAP, xpath); } - return getConfig(solrConfig, node.getNodeName(),DOMUtil.toMap(node.getAttributes()), xpath); + return getConfig(solrConfig, node.name(), node.attributes().asMap(), xpath); } @SuppressWarnings({"unchecked"}) - public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map attrs, String xpath) { + public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map attrs, String xpath) { CacheConfig config = new CacheConfig(); config.nodeName = nodeName; @SuppressWarnings({"rawtypes"}) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index a364757b8af6..00ecefb5b003 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.lucene.analysis.Analyzer; @@ -31,6 +30,7 @@ import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.search.Sort; import org.apache.lucene.util.InfoStream; +import org.apache.solr.common.ConfigNode; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.DirectoryFactory; @@ -91,11 +91,12 @@ public class SolrIndexConfig implements MapSerializable { public final PluginInfo mergedSegmentWarmerInfo; public InfoStream infoStream = InfoStream.NO_OUTPUT; + private ConfigNode node; /** * Internal constructor for setting defaults based on Lucene Version */ - private SolrIndexConfig(SolrConfig solrConfig) { + private SolrIndexConfig() { useCompoundFile = false; maxBufferedDocs = -1; ramBufferSizeMB = 100; @@ -109,88 +110,78 @@ private SolrIndexConfig(SolrConfig solrConfig) { // enable coarse-grained metrics by default metricsInfo = new PluginInfo("metrics", Collections.emptyMap(), null, null); } - + private ConfigNode get(String s) { return node.get(s); } + public SolrIndexConfig(SolrConfig cfg, SolrIndexConfig def) { + this(cfg.get("indexConfig"), def); + } /** * Constructs a SolrIndexConfig which parses the Lucene related config params in solrconfig.xml - * @param solrConfig the overall SolrConfig object - * @param prefix the XPath prefix for which section to parse (mandatory) * @param def a SolrIndexConfig instance to pick default values from (optional) */ - public SolrIndexConfig(SolrConfig solrConfig, String prefix, SolrIndexConfig def) { - if (prefix == null) { - prefix = "indexConfig"; - log.debug("Defaulting to prefix '{}' for index configuration", prefix); - } - + public SolrIndexConfig(ConfigNode cfg, SolrIndexConfig def) { + this.node = cfg; if (def == null) { - def = new SolrIndexConfig(solrConfig); + def = new SolrIndexConfig(); } + // sanity check: this will throw an error for us if there is more then one // config section - Object unused = solrConfig.getNode(prefix, false); +// Object unused = solrConfig.getNode(prefix, false); // Assert that end-of-life parameters or syntax is not in our config. // Warn for luceneMatchVersion's before LUCENE_3_6, fail fast above assertWarnOrFail("The myclass syntax is no longer supported in solrconfig.xml. Please use syntax instead.", - !((solrConfig.getNode(prefix + "/mergeScheduler", false) != null) && (solrConfig.get(prefix + "/mergeScheduler/@class", null) == null)), + get("mergeScheduler").isNull() || get("mergeScheduler").attr("class") != null, true); assertWarnOrFail("Beginning with Solr 7.0, myclass is no longer supported, use instead.", - !((solrConfig.getNode(prefix + "/mergePolicy", false) != null) && (solrConfig.get(prefix + "/mergePolicy/@class", null) == null)), + get("mergePolicy").isNull() || get("mergePolicy").attr("class") != null, true); assertWarnOrFail("The true|false parameter is no longer valid in solrconfig.xml.", - solrConfig.get(prefix + "/luceneAutoCommit", null) == null, + get("luceneAutoCommit").isNull(), true); - useCompoundFile = solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile); - maxBufferedDocs = solrConfig.getInt(prefix+"/maxBufferedDocs", def.maxBufferedDocs); - ramBufferSizeMB = solrConfig.getDouble(prefix+"/ramBufferSizeMB", def.ramBufferSizeMB); - maxCommitMergeWaitMillis = solrConfig.getInt(prefix+"/maxCommitMergeWaitTime", def.maxCommitMergeWaitMillis); + useCompoundFile = get("useCompoundFile").boolVal(def.useCompoundFile); + maxBufferedDocs = get("maxBufferedDocs").intVal(def.maxBufferedDocs); + ramBufferSizeMB = get("ramBufferSizeMB").doubleVal(def.ramBufferSizeMB); + maxCommitMergeWaitMillis = get("maxCommitMergeWaitTime").intVal(def.maxCommitMergeWaitMillis); // how do we validate the value?? - ramPerThreadHardLimitMB = solrConfig.getInt(prefix+"/ramPerThreadHardLimitMB", def.ramPerThreadHardLimitMB); + ramPerThreadHardLimitMB = get("ramPerThreadHardLimitMB").intVal(def.ramPerThreadHardLimitMB); - writeLockTimeout=solrConfig.getInt(prefix+"/writeLockTimeout", def.writeLockTimeout); - lockType=solrConfig.get(prefix+"/lockType", def.lockType); + writeLockTimeout= get("writeLockTimeout").intVal(def.writeLockTimeout); + lockType = get("lockType").txt(def.lockType); - List infos = solrConfig.readPluginInfos(prefix + "/metrics", false, false); - if (infos.isEmpty()) { - metricsInfo = def.metricsInfo; - } else { - metricsInfo = infos.get(0); - } - mergeSchedulerInfo = getPluginInfo(prefix + "/mergeScheduler", solrConfig, def.mergeSchedulerInfo); - mergePolicyFactoryInfo = getPluginInfo(prefix + "/mergePolicyFactory", solrConfig, def.mergePolicyFactoryInfo); + metricsInfo = getPluginInfo(get("metrics"), def.metricsInfo); + mergeSchedulerInfo = getPluginInfo(get("mergeScheduler"), def.mergeSchedulerInfo); + mergePolicyFactoryInfo = getPluginInfo(get("mergePolicyFactory"), def.mergePolicyFactoryInfo); assertWarnOrFail("Beginning with Solr 7.0, is no longer supported, use instead.", - getPluginInfo(prefix + "/mergePolicy", solrConfig, null) == null, + get("mergePolicy").isNull(), true); assertWarnOrFail("Beginning with Solr 7.0, is no longer supported, configure it on the relevant instead.", - solrConfig.getInt(prefix+"/maxMergeDocs", 0) == 0, + get("maxMergeDocs").isNull(), true); assertWarnOrFail("Beginning with Solr 7.0, is no longer supported, configure it on the relevant instead.", - solrConfig.getInt(prefix+"/mergeFactor", 0) == 0, + get("maxMergeFactor").isNull(), true); - String val = solrConfig.get(prefix + "/termIndexInterval", null); - if (val != null) { + if (get("termIndexInterval").exists()) { throw new IllegalArgumentException("Illegal parameter 'termIndexInterval'"); } - boolean infoStreamEnabled = solrConfig.getBool(prefix + "/infoStream", false); - if(infoStreamEnabled) { - String infoStreamFile = solrConfig.get(prefix + "/infoStream/@file", null); - if (infoStreamFile == null) { + if(get("infoStream").boolVal(false)) { + if (get("infoStream").attr("file") == null) { log.info("IndexWriter infoStream solr logging is enabled"); infoStream = new LoggingInfoStream(); } else { throw new IllegalArgumentException("Remove @file from to output messages to solr's logfile"); } } - mergedSegmentWarmerInfo = getPluginInfo(prefix + "/mergedSegmentWarmer", solrConfig, def.mergedSegmentWarmerInfo); + mergedSegmentWarmerInfo = getPluginInfo(get("mergedSegmentWarmer"), def.mergedSegmentWarmerInfo); assertWarnOrFail("Beginning with Solr 5.0, option is no longer supported and should be removed from solrconfig.xml (these integrity checks are now automatic)", - (null == solrConfig.getNode(prefix + "/checkIntegrityAtMerge", false)), + get( "checkIntegrityAtMerge").isNull(), true); } @@ -215,9 +206,10 @@ public Map toMap(Map map) { return m; } - private PluginInfo getPluginInfo(String path, SolrConfig solrConfig, PluginInfo def) { - List l = solrConfig.readPluginInfos(path, false, true); - return l.isEmpty() ? def : l.get(0); + private PluginInfo getPluginInfo(ConfigNode node , PluginInfo def) { + return node != null && node.exists() ? + new PluginInfo(node, "[solrconfig.xml] " + node.name(), false, false) : + def; } private static class DelayedSchemaAnalyzer extends DelegatingAnalyzerWrapper { diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index b97f8129ae78..bf21ff4d3937 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -94,8 +94,8 @@ public VersionInfo(UpdateLog ulog, int nBuckets) { this.ulog = ulog; IndexSchema schema = ulog.uhandler.core.getLatestSchema(); versionField = getAndCheckVersionField(schema); - versionBucketLockTimeoutMs = ulog.uhandler.core.getSolrConfig().getInt("updateHandler/versionBucketLockTimeoutMs", - Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0"))); + versionBucketLockTimeoutMs = ulog.uhandler.core.getSolrConfig().get("updateHandler").get("versionBucketLockTimeoutMs") + .intVal(Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0"))); buckets = new VersionBucket[ BitUtil.nextHighestPowerOfTwo(nBuckets) ]; for (int i=0; i 0) { diff --git a/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java b/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java index 6afc5a542534..fb54af1eb08e 100644 --- a/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java +++ b/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java @@ -18,7 +18,9 @@ package org.apache.solr.util; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Function; import org.apache.solr.cluster.api.SimpleMap; @@ -43,7 +45,7 @@ public String name() { } @Override - public String textValue() { + public String txt() { return DOMUtil.getText(node); } @@ -54,7 +56,8 @@ public DOMConfigNode(Node node) { @Override public SimpleMap attributes() { if (attrs != null) return attrs; - return attrs = new WrappedSimpleMap<>(DOMUtil.toMap(node.getAttributes())); + Map attrs = DOMUtil.toMap(node.getAttributes()); + return this.attrs = attrs.size() == 0 ? EMPTY : new WrappedSimpleMap<>(attrs); } @Override @@ -64,7 +67,7 @@ public ConfigNode child(String name) { } @Override - public List children(String name) { + public List getAll(String name) { List result = new ArrayList<>(); forEachChild(it -> { if (name.equals(it.name())) { @@ -85,5 +88,6 @@ public void forEachChild(Function fun) { if (Boolean.FALSE == toContinue) break; } } + private static final SimpleMap EMPTY = new WrappedSimpleMap<>(Collections.emptyMap()); } diff --git a/solr/core/src/java/org/apache/solr/util/DataConfigNode.java b/solr/core/src/java/org/apache/solr/util/DataConfigNode.java index 38940eec4d0d..78676a2951d5 100644 --- a/solr/core/src/java/org/apache/solr/util/DataConfigNode.java +++ b/solr/core/src/java/org/apache/solr/util/DataConfigNode.java @@ -19,7 +19,7 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -27,56 +27,64 @@ import java.util.function.Function; import java.util.function.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.solr.cluster.api.SimpleMap; import org.apache.solr.common.ConfigNode; import org.apache.solr.common.util.PropertiesUtil; +import org.apache.solr.common.util.WrappedSimpleMap; /** * ConfigNode impl that copies and maintains data internally from DOM */ public class DataConfigNode implements ConfigNode { - final String name; - final SimpleMap attributes; - private final Map> kids = new HashMap<>(); - private final String textData; + public final String name; + public final SimpleMap attributes; + public final SimpleMap> kids ; + public final String textData; + public DataConfigNode(ConfigNode root) { + Map> kids = new LinkedHashMap<>(); name = root.name(); attributes = wrap(root.attributes()); - textData = root.textValue(); + textData = root.txt(); root.forEachChild(it -> { List nodes = kids.computeIfAbsent(it.name(), k -> new ArrayList<>()); - - nodes.add(new DataConfigNode(it)); + nodes.add(new DataConfigNode(it)); return Boolean.TRUE; }); - + for (Map.Entry> e : kids.entrySet()) { + if(e.getValue() != null) { + e.setValue(ImmutableList.copyOf(e.getValue())); + } + } + this.kids = kids.isEmpty()? EMPTY: new WrappedSimpleMap<>(ImmutableMap.copyOf(kids)); } public String subtituteVal(String s) { - Function props = SUBSTITUTES.get(); - if (props == null) return s; - return PropertiesUtil.substitute(s, props); + return PropertiesUtil.substitute(s, SUBSTITUTES.get()); } private SimpleMap wrap(SimpleMap delegate) { + if(delegate.size() == 0) return delegate;//avoid unnecessary object creation return new SimpleMap<>() { - @Override - public String get(String key) { - return subtituteVal(delegate.get(key)); - } + @Override + public String get(String key) { + return subtituteVal(delegate.get(key)); + } - @Override - public void forEachEntry(BiConsumer fun) { - delegate.forEachEntry((k, v) -> fun.accept(k, subtituteVal(v))); - } + @Override + public void forEachEntry(BiConsumer fun) { + delegate.forEachEntry((k, v) -> fun.accept(k, subtituteVal(v))); + } - @Override - public int size() { - return delegate.size(); - } - }; + @Override + public int size() { + return delegate.size(); + } + }; } @Override @@ -85,8 +93,8 @@ public String name() { } @Override - public String textValue() { - return subtituteVal(textData); + public String txt() { + return subtituteVal(textData); } @Override @@ -101,12 +109,12 @@ public ConfigNode child(String name) { } @Override - public List children(String name) { - return kids.getOrDefault(name, Collections.emptyList()); + public List getAll(String name) { + return kids.get(name, Collections.emptyList()); } @Override - public List children(Predicate test, Set matchNames) { + public List getAll(Predicate test, Set matchNames) { List result = new ArrayList<>(); for (String s : matchNames) { List vals = kids.get(s); @@ -123,10 +131,11 @@ public List children(Predicate test, Set matchNa @Override public void forEachChild(Function fun) { - kids.forEach((s, configNodes) -> { + kids.forEachEntry((s, configNodes) -> { if (configNodes != null) { configNodes.forEach(fun::apply); } }); } + public static final SimpleMap> EMPTY = new WrappedSimpleMap<>(Collections.emptyMap()); } diff --git a/solr/core/src/resources/EditableSolrConfigAttributes.json b/solr/core/src/resources/EditableSolrConfigAttributes.json index 03bb1b67d90a..fe06fa0cc191 100644 --- a/solr/core/src/resources/EditableSolrConfigAttributes.json +++ b/solr/core/src/resources/EditableSolrConfigAttributes.json @@ -56,10 +56,6 @@ "enableLazyFieldLoading":1, "boolTofilterOptimizer":1, "maxBooleanClauses":1}, - "jmx":{ - "agentId":0, - "serviceUrl":0, - "rootName":0}, "requestDispatcher":{ "handleSelect":0, "requestParsers":{ diff --git a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java index 91fd9ae3fde5..677ab39c1f6b 100644 --- a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java @@ -16,6 +16,8 @@ */ package org.apache.solr.core; +import org.junit.Ignore; + public class TestBadConfig extends AbstractBadConfigTestBase { public void testUnsetSysProperty() throws Exception { @@ -26,14 +28,17 @@ public void testNRTModeProperty() throws Exception { assertConfigs("bad-solrconfig-nrtmode.xml","schema.xml", "nrtMode"); } + @Ignore public void testMultipleDirectoryFactories() throws Exception { assertConfigs("bad-solrconfig-multiple-dirfactory.xml", "schema12.xml", "directoryFactory"); } + @Ignore public void testMultipleIndexConfigs() throws Exception { assertConfigs("bad-solrconfig-multiple-indexconfigs.xml", "schema12.xml", "indexConfig"); } + @Ignore public void testMultipleCFS() throws Exception { assertConfigs("bad-solrconfig-multiple-cfs.xml", "schema12.xml", "useCompoundFile"); diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index cd86d4fdc98b..f497d14e5941 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -200,9 +200,9 @@ public void testCompressionModeDefault() throws IOException { SolrCore c = null; SolrConfig config = TestHarness.createConfig(testSolrHome, previousCoreName, "solrconfig_codec2.xml"); - assertEquals("Unexpected codec factory for this test.", "solr.SchemaCodecFactory", config.get("codecFactory/@class")); - assertNull("Unexpected configuration of codec factory for this test. Expecting empty element", - config.getNode("codecFactory", false).getFirstChild()); + assertEquals("Unexpected codec factory for this test.", "solr.SchemaCodecFactory", config.get("codecFactory").attr("class")); + assertTrue("Unexpected configuration of codec factory for this test. Expecting empty element", + config.get("codecFactory").getAll(null, (String)null).isEmpty()); IndexSchema schema = IndexSchemaFactory.buildIndexSchema("schema_codec.xml", config); CoreContainer coreContainer = h.getCoreContainer(); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java b/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java new file mode 100644 index 000000000000..ab85f399afc2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.util.SuppressForbidden; +import org.apache.solr.util.ExternalPaths; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; + +import static org.apache.solr.core.TestConfigSets.solrxml; + +public class TestConfLoadPerf extends SolrTestCaseJ4 { + + @Ignore + @SuppressForbidden(reason = "Needed to provide time for tests.") + public void testPerf() throws Exception{ + String sourceHome = ExternalPaths.SOURCE_HOME; + File configSetDir = new File(sourceHome, "server/solr/configsets/sample_techproducts_configs/conf"); + + String configSetsBaseDir = TEST_PATH().resolve("configsets").toString(); + + + File file = new File(configSetDir, "solrconfig.xml"); + byte[] b = new byte[(int) file.length()]; + new FileInputStream(file).read(b); + + Path testDirectory = createTempDir(); + + System.setProperty("configsets", configSetsBaseDir); + + CoreContainer container = new CoreContainer(SolrXmlConfig.fromString(testDirectory, solrxml)); + container.load(); + container.shutdown(); + + SolrResourceLoader srl = new SolrResourceLoader("temp", Collections.emptyList(), container.solrHome, container.getResourceLoader().classLoader){ + + @Override + public CoreContainer getCoreContainer() { + return container; + } + + @Override + public InputStream openResource(String resource) throws IOException { + if(resource.equals("solrconfig.xml")) { + Stat stat = new Stat(); + stat.setVersion(1); + return new ZkSolrResourceLoader.ZkByteArrayInputStream(b, file.getAbsolutePath(), stat); + } else { + throw new FileNotFoundException(resource); + } + + } + }; + System.gc(); + long heapSize = Runtime.getRuntime().totalMemory(); + List allConfigs = new ArrayList<>(); + long startTime = System.currentTimeMillis(); + for(int i=0;i<100;i++) { + allConfigs.add(SolrConfig.readFromResourceLoader(srl, "solrconfig.xml", true, null)); + + } + System.gc(); + System.out.println("TIME_TAKEN : "+(System.currentTimeMillis()-startTime)); + System.out.println("HEAP_SIZE : "+((Runtime.getRuntime().totalMemory()-heapSize)/(1024))); + } +} diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java index ccf31144b368..567262aec95d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java @@ -16,17 +16,18 @@ */ package org.apache.solr.core; -import javax.xml.xpath.XPathConstants; import java.io.IOException; import java.io.InputStream; import java.util.LinkedHashMap; import java.util.Collections; +import java.util.List; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.util.InfoStream; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.ConfigNode; import org.apache.solr.handler.admin.ShowFileRequestHandler; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchemaFactory; @@ -35,8 +36,6 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; public class TestConfig extends SolrTestCaseJ4 { @@ -80,24 +79,24 @@ public void testDisableRequetsHandler() throws Exception { public void testJavaProperty() { // property values defined in build.xml - String s = solrConfig.get("propTest"); + String s = solrConfig.get("propTest").txt(); assertEquals("prefix-proptwo-suffix", s); - s = solrConfig.get("propTest/@attr1", "default"); + s = solrConfig.get("propTest").attr("attr1", "default"); assertEquals("propone-${literal}", s); - s = solrConfig.get("propTest/@attr2", "default"); + s = solrConfig.get("propTest").attr("attr2", "default"); assertEquals("default-from-config", s); - s = solrConfig.get("propTest[@attr2='default-from-config']", "default"); - assertEquals("prefix-proptwo-suffix", s); - NodeList nl = (NodeList) solrConfig.evaluate("propTest", XPathConstants.NODESET); - assertEquals(1, nl.getLength()); - assertEquals("prefix-proptwo-suffix", nl.item(0).getTextContent()); + assertEquals("prefix-proptwo-suffix", solrConfig.get("propTest", + it -> "default-from-config".equals(it.attr("attr2"))).txt()); + + List nl = solrConfig.root.getAll("propTest"); + assertEquals(1, nl.size()); + assertEquals("prefix-proptwo-suffix", nl.get(0).txt()); - Node node = solrConfig.getNode("propTest", true); - assertEquals("prefix-proptwo-suffix", node.getTextContent()); + assertEquals("prefix-proptwo-suffix", solrConfig.get("propTest").txt()); } // sometime if the config referes to old things, it must be replaced with new stuff diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java index 07708c66d75f..c782be396275 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigOverlay.java @@ -46,9 +46,6 @@ public void testPaths() { assertTrue(isEditableProp("query.queryResultMaxDocsCached", false, null)); assertTrue(isEditableProp("query.enableLazyFieldLoading", false, null)); assertTrue(isEditableProp("query.boolTofilterOptimizer", false, null)); - assertTrue(isEditableProp("jmx.agentId", false, null)); - assertTrue(isEditableProp("jmx.serviceUrl", false, null)); - assertTrue(isEditableProp("jmx.rootName", false, null)); assertTrue(isEditableProp("requestDispatcher.requestParsers.multipartUploadLimitInKB", false, null)); assertTrue(isEditableProp("requestDispatcher.requestParsers.formdataUploadLimitInKB", false, null)); diff --git a/solr/core/src/test/org/apache/solr/core/TestSimpleTextCodec.java b/solr/core/src/test/org/apache/solr/core/TestSimpleTextCodec.java index 9f6776b8fdec..abc7723e382d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSimpleTextCodec.java +++ b/solr/core/src/test/org/apache/solr/core/TestSimpleTextCodec.java @@ -33,7 +33,7 @@ public static void beforeClass() throws Exception { public void test() throws Exception { SolrConfig config = h.getCore().getSolrConfig(); - String codecFactory = config.get("codecFactory/@class"); + String codecFactory = config.get("codecFactory").attr("class"); assertEquals("Unexpected solrconfig codec factory", "solr.SimpleTextCodecFactory", codecFactory); assertEquals("Unexpected core codec", "SimpleText", h.getCore().getCodec().getName()); diff --git a/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java index b15e167aa058..a36319029ab5 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java @@ -204,14 +204,25 @@ public void testDefaultBuildOnStartupNotStoredDict() throws Exception { final String suggester = "suggest_doc_default_startup_no_store"; // validate that this suggester is not storing the lookup - assertEquals(suggester, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='name']", false)); - assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='storeDir']", false)); + assertEquals(suggester, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",7). + get("str", n -> "name".equals(n.attr("name"))).txt()); + + assertNull( h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",7). + get("str", n -> "storeDir".equals(n.attr("name"))).txt()); // validate that this suggester only builds manually and has not buildOnStartup parameter - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='buildOnCommit']", true)); - assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[8]/str[@name='buildOnStartup']", false)); + + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",7). + get("str", n -> "buildOnCommit".equals(n.attr("name"))).txt()); + + assertNull(h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",7). + get("str", n -> "buildOnStartup".equals(n.attr("name"))).txt()); reloadCore(random().nextBoolean()); @@ -254,15 +265,27 @@ public void testDefaultBuildOnStartupStoredDict() throws Exception { final String suggester = "suggest_doc_default_startup"; // validate that this suggester is storing the lookup - assertEquals(suggester, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='name']", false)); - assertEquals(suggester, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='storeDir']", false)); + + assertEquals(suggester, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",6) + .get("str", n -> "name".equals(n.attr("name"))).txt()); + + assertEquals(suggester, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",6) + .get("str", n -> "storeDir".equals(n.attr("name"))).txt()); // validate that this suggester only builds manually and has not buildOnStartup parameter - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='buildOnCommit']", true)); - assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[7]/str[@name='buildOnStartup']", false)); + + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",6) + .get("str", n -> "buildOnCommit".equals(n.attr("name"))).txt()); + + assertNull( h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",6) + .get("str", n -> "buildOnStartup".equals(n.attr("name"))).txt()); assertQ(req("qt", rh, SuggesterParams.SUGGEST_DICT, suggester, @@ -339,16 +362,27 @@ public void testLoadOnStartup() throws Exception { final String suggester = "suggest_fuzzy_doc_manal_build"; // validate that this suggester is storing the lookup - assertEquals(suggester, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='name']", false)); - assertEquals(suggester, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='storeDir']", false)); + + assertEquals(suggester, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",5). + get("str", n -> "name".equals(n.attr("name"))).txt()); + + assertEquals(suggester, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",5). + get("str", n -> "storeDir".equals(n.attr("name"))).txt()); // validate that this suggester only builds manually - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='buildOnCommit']", true)); - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[6]/str[@name='buildOnStartup']", true)); + + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",5). + get("str", n -> "buildOnCommit".equals(n.attr("name"))).txt()); + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",5). + get("str", n -> "buildOnStartup".equals(n.attr("name"))).txt()); // build the suggester manually assertQ(req("qt", rh, @@ -402,15 +436,25 @@ private void doTestBuildOnStartup(boolean createNewCores) throws Exception { h.getCore().getSolrConfig().useColdSearcher); // validate that this suggester is not storing the lookup and buildOnStartup is not set - assertEquals(suggesterFuzzy, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='name']", false)); - assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='storeDir']", false)); + assertEquals(suggesterFuzzy, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",2). + get("str", n -> "name".equals(n.attr("name"))).txt()); + + assertNull(h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",2) + .get("str", n -> "storeDir".equals(n.attr("name"))).txt()); + // assert that buildOnStartup=false - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='buildOnStartup']", false)); - assertEquals("true", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[3]/str[@name='buildOnCommit']", false)); + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",2) + .get("str", n -> "buildOnStartup".equals(n.attr("name"))).txt()); + assertEquals("true", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",2). + get("str", n -> "buildOnCommit".equals(n.attr("name"))).txt()); // verify that this suggester is built (there was a commit in setUp) assertQ(req("qt", rh, @@ -453,13 +497,21 @@ private void doTestBuildOnStartup(boolean createNewCores) throws Exception { final String suggestStartup = "suggest_fuzzy_doc_dict_build_startup"; // repeat the test with "suggest_fuzzy_doc_dict_build_startup", it is exactly the same but with buildOnStartup=true - assertEquals(suggestStartup, - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='name']", false)); - assertNull(h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='storeDir']", false)); - assertEquals("true", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='buildOnStartup']", false)); - assertEquals("false", - h.getCore().getSolrConfig().getVal("//searchComponent[@name='suggest']/lst[5]/str[@name='buildOnCommit']", false)); + assertEquals(suggestStartup, + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",4) + .get("str", n -> "name".equals(n.attr("name"))).txt()); + assertNull( h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",4) + .get("str", n -> "storeDir".equals(n.attr("name"))).txt()); + assertEquals("true", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",4) + .get("str", n -> "buildOnStartup".equals(n.attr("name"))).txt()); + assertEquals("false", + h.getCore().getSolrConfig().get("searchComponent", n -> "suggest".equals(n.attr("name"))) + .get("lst",4) + .get("str", n -> "buildOnCommit".equals(n.attr("name"))).txt()); // reload the core reloadCore(createNewCores); diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index 4f43b10a2605..221dd07e4700 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -70,7 +70,7 @@ public void tearDown() throws Exception { @Test public void testFailingSolrIndexConfigCreation() throws Exception { SolrConfig solrConfig = new SolrConfig(instanceDir,"bad-mpf-solrconfig.xml"); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); h.getCore().setLatestSchema(indexSchema); @@ -83,7 +83,7 @@ public void testFailingSolrIndexConfigCreation() throws Exception { public void testTieredMPSolrIndexConfigCreation() throws Exception { String solrConfigFileName = solrConfigFileNameTieredMergePolicyFactory; SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileName); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); h.getCore().setLatestSchema(indexSchema); @@ -108,7 +108,7 @@ public void testTieredMPSolrIndexConfigCreation() throws Exception { public void testConcurrentMergeSchedularSolrIndexConfigCreation() throws Exception { String solrConfigFileName = solrConfigFileNameConnMSPolicyFactory; SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileName); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); h.getCore().setLatestSchema(indexSchema); @@ -132,7 +132,7 @@ public void testSortingMPSolrIndexConfigCreation() throws Exception { final boolean expectedFieldSortDescending = true; SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSortingMergePolicyFactory); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); assertNotNull(solrIndexConfig); IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); @@ -150,7 +150,7 @@ public void testSortingMPSolrIndexConfigCreation() throws Exception { public void testMergedSegmentWarmerIndexConfigCreation() throws Exception { SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameWarmerRandomMergePolicyFactory); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); assertNotNull(solrIndexConfig); assertNotNull(solrIndexConfig.mergedSegmentWarmerInfo); assertEquals(SimpleMergedSegmentWarmer.class.getName(), @@ -166,7 +166,7 @@ public void testToMap() throws Exception { final String solrConfigFileNameTMP = solrConfigFileNameTieredMergePolicyFactory; final String solrConfigFileName = (random().nextBoolean() ? solrConfigFileNameWarmer : solrConfigFileNameTMP); SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileName); - SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); assertNotNull(solrIndexConfig); assertNotNull(solrIndexConfig.mergePolicyFactoryInfo); if (solrConfigFileName.equals(solrConfigFileNameWarmerRandomMergePolicyFactory)) { diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java b/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java index 81da17197a88..885b11d881c8 100644 --- a/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java +++ b/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java @@ -20,6 +20,8 @@ import org.apache.solr.common.MapWriter; import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -82,4 +84,13 @@ public void accept(String k, T v) { default void writeMap(EntryWriter ew) throws IOException { forEachEntry(ew::putNoEx); } + + default Map asMap( Map sink) { + forEachEntry(sink::put); + return sink; + } + + default Map asMap() { + return asMap(new LinkedHashMap()); + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/ConfigNode.java b/solr/solrj/src/java/org/apache/solr/common/ConfigNode.java index 1a67b5232caf..03dba324611f 100644 --- a/solr/solrj/src/java/org/apache/solr/common/ConfigNode.java +++ b/solr/solrj/src/java/org/apache/solr/common/ConfigNode.java @@ -17,16 +17,23 @@ package org.apache.solr.common; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import org.apache.solr.cluster.api.SimpleMap; +import org.apache.solr.common.util.WrappedSimpleMap; + +import static org.apache.solr.common.ConfigNode.Helpers.*; /** * A generic interface that represents a config file, mostly XML + * Please note that this is an immutable, read-only object. */ public interface ConfigNode { ThreadLocal> SUBSTITUTES = new ThreadLocal<>(); @@ -36,11 +43,6 @@ public interface ConfigNode { */ String name(); - /** - * Text value of the node - */ - String textValue(); - /** * Attributes */ @@ -53,6 +55,56 @@ default ConfigNode child(String name) { return child(null, name); } + /** + * Child by name or return an empty node if null + * if there are multiple values , it returns the first elem + * This never returns a null + */ + default ConfigNode get(String name) { + ConfigNode child = child(null, name); + return child == null? EMPTY: child; + } + default ConfigNode get(String name, Predicate test) { + List children = getAll(test, name); + if(children.isEmpty()) return EMPTY; + return children.get(0); + } + default ConfigNode get(String name, int idx) { + List children = getAll(null, name); + if(idx < children.size()) return children.get(idx); + return EMPTY; + + } + + default ConfigNode child(List path) { + ConfigNode node = this; + for (String s : path) { + node = node.child(s); + if (node == null) break; + } + return node; + } + + default ConfigNode child(String name, Supplier err) { + ConfigNode n = child(name); + if(n == null) throw err.get(); + return n; + } + + default boolean boolVal(boolean def) { return _bool(txt(),def); } + default int intVal(int def) { return _int(txt(), def); } + default String attr(String name, String def) { return _txt(attributes().get(name), def);} + default String attr(String name) { return attributes().get(name);} + default String requiredStrAttr(String name, Supplier err) { + String attr = attr(name); + if(attr == null && err != null) throw err.get(); + return attr; + } + default int intAttr(String name, int def) { return _int(attr(name), def); } + default boolean boolAttr(String name, boolean def){ return _bool(attr(name), def); } + default String txt(String def) { return txt() == null ? def : txt();} + String txt() ; + default double doubleVal(double def){ return _double(txt(), def); } /**Iterate through child nodes with the name and return the first child that matches */ default ConfigNode child(Predicate test, String name) { @@ -72,15 +124,15 @@ default ConfigNode child(Predicate test, String name) { * @param nodeNames names of tags to be returned * @param test check for the nodes to be returned */ - default List children(Predicate test, String... nodeNames) { - return children(test, nodeNames == null ? Collections.emptySet() : Set.of(nodeNames)); + default List getAll(Predicate test, String... nodeNames) { + return getAll(test, nodeNames == null ? Collections.emptySet() : new HashSet<>(Arrays.asList(nodeNames))); } /**Iterate through child nodes with the names and return all the matching children * @param matchNames names of tags to be returned * @param test check for the nodes to be returned */ - default List children(Predicate test, Set matchNames) { + default List getAll(Predicate test, Set matchNames) { List result = new ArrayList<>(); forEachChild(it -> { if (matchNames != null && !matchNames.isEmpty() && !matchNames.contains(it.name())) return Boolean.TRUE; @@ -90,15 +142,76 @@ default List children(Predicate test, Set matchN return result; } - default List children(String name) { - return children(null, Collections.singleton(name)); + default List getAll(String name) { + return getAll(null, Collections.singleton(name)); } + default boolean exists() { return true; } + default boolean isNull() { return false; } + /** abortable iterate through children * * @param fun consume the node and return true to continue or false to abort */ void forEachChild(Function fun); + /**An empty node object. + * usually returned when the node is absent + * + */ + ConfigNode EMPTY = new ConfigNode() { + @Override + public String name() { + return null; + } + + @Override + public String txt() { return null; } + + @Override + public SimpleMap attributes() { + return empty_attrs; + } + + @Override + public String attr(String name) { return null; } + @Override + public String attr(String name, String def) { return def; } + + @Override + public ConfigNode child(String name) { return null; } + + @Override + public ConfigNode get(String name) { + return EMPTY; + } + + public boolean exists() { return false; } + + @Override + public boolean isNull() { return true; } + + @Override + public void forEachChild(Function fun) { } + } ; + SimpleMap empty_attrs = new WrappedSimpleMap<>(Collections.emptyMap()); + + class Helpers { + static boolean _bool(Object v, boolean def) { return v == null ? def : Boolean.parseBoolean(v.toString()); } + static String _txt(Object v, String def) { return v == null ? def : v.toString(); } + static int _int(Object v, int def) { return v==null? def: Integer.parseInt(v.toString()); } + static double _double(Object v, double def) { return v == null ? def: Double.parseDouble(v.toString()); } + public static Predicate at(int i) { + return new Predicate() { + int index =0; + @Override + public boolean test(ConfigNode node) { + if(index == i) return true; + index++; + return false; + } + }; + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java index d0b1f7b94e90..dc1e22fb3145 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java @@ -212,6 +212,8 @@ private static Object parseVal(String type, String name, String textValue) { val = Double.valueOf(textValue); } else if ("bool".equals(type)) { val = StrUtils.parseBool(textValue); + } else if("null".equals(type)) { + val = null; } // :NOTE: Unexpected Node names are ignored // :TODO: should we generate an error here? @@ -231,7 +233,7 @@ public static NamedList readNamedListChildren(ConfigNode configNode) { String tag = it.name(); String varName = it.attributes().get("name"); if (NL_TAGS.contains(tag)) { - result.add(varName, parseVal(tag, varName, it.textValue())); + result.add(varName, parseVal(tag, varName, it.txt())); } if ("lst".equals(tag)) { result.add(varName, readNamedListChildren(it)); @@ -240,7 +242,9 @@ public static NamedList readNamedListChildren(ConfigNode configNode) { result.add(varName, l); it.forEachChild(n -> { if (NL_TAGS.contains(n.name())) { - l.add(parseVal(n.name(), null, n.textValue())); + l.add(parseVal(n.name(), null, n.txt())); + } else if("lst".equals(n.name())){ + l.add(readNamedListChildren(n)); } return Boolean.TRUE; }); diff --git a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java b/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java index 1fc6afc36482..6581c83041b7 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java @@ -19,6 +19,7 @@ import org.apache.solr.cluster.api.SimpleMap; import java.util.LinkedHashMap; +import java.util.Map; import java.util.function.BiConsumer; public class LinkedSimpleHashMap extends LinkedHashMap implements SimpleMap { @@ -31,4 +32,9 @@ public T get(String key) { public void forEachEntry(BiConsumer fun) { } + + @Override + public Map asMap() { + return this; + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java b/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java index e8f58a5be98a..ccf000b9bb03 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java @@ -19,6 +19,7 @@ import org.apache.solr.cluster.api.SimpleMap; +import java.util.Collections; import java.util.Map; import java.util.function.BiConsumer; @@ -46,4 +47,14 @@ public WrappedSimpleMap(Map delegate) { this.delegate = delegate; } + @Override + public Map asMap(Map sink) { + sink.putAll(delegate); + return sink; + } + + @Override + public Map asMap() { + return Collections.unmodifiableMap(delegate); + } } From 1ce541429591cba4e615effded06345d12d3e4a1 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 29 Apr 2021 23:20:54 +1000 Subject: [PATCH 2/3] refactored --- .../apache/solr/core/OverlaidConfigNode.java | 92 +++++++++++++++++++ .../java/org/apache/solr/core/SolrConfig.java | 80 +--------------- .../org/apache/solr/common/util/DOMUtil.java | 4 +- 3 files changed, 96 insertions(+), 80 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java diff --git a/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java b/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java new file mode 100644 index 000000000000..777078994388 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java @@ -0,0 +1,92 @@ +package org.apache.solr.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; + +import org.apache.solr.cluster.api.SimpleMap; +import org.apache.solr.common.ConfigNode; + +/**A config node impl which has an overlay + * + */ +class OverlaidConfigNode implements ConfigNode { + + private final ConfigOverlay overlay; + private final String _name; + private final ConfigNode delegate; + private final OverlaidConfigNode parent; + + OverlaidConfigNode(ConfigOverlay overlay, String name, OverlaidConfigNode parent, ConfigNode delegate) { + this.overlay = overlay; + this._name = name; + this.delegate = delegate; + this.parent = parent; + } + + private List path(List path) { + if(path== null) path = new ArrayList<>(5); + try { + if (parent != null) return parent.path(path); + } finally { + path.add(_name); + } + return path; + } + + @Override + public ConfigNode get(String name) { + return wrap(delegate.get(name), name); + } + + private ConfigNode wrap(ConfigNode n, String name) { + return new OverlaidConfigNode(overlay, name,this, n); + } + + @Override + public ConfigNode get(String name, Predicate test) { + return wrap(delegate.get(name, test), name); + } + + @Override + public String txt() { + return overlayText(delegate.txt(), null); + } + + @Override + public ConfigNode get(String name, int idx) { + return wrap(delegate.get(name, idx), name); + } + + @Override + public String name() { + return delegate.name(); + } + @Override + public SimpleMap attributes() { + return delegate.attributes(); + } + + @Override + public boolean exists() { + return delegate.exists(); + } + + @Override + public String attr(String name) { + return overlayText(delegate.attr(name),name); + } + + private String overlayText(String def, String appendToPath) { + List path = path(null); + if(appendToPath !=null) path.add(appendToPath); + Object val = overlay.getXPathProperty(path); + return val ==null? def: val.toString(); + } + + @Override + public void forEachChild(Function fun) { + delegate.forEachChild(fun); + } +} diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 9fafda4edbe5..92312864d482 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -54,7 +54,6 @@ import org.apache.solr.client.solrj.io.stream.expr.Expressible; import org.apache.solr.cloud.RecoveryStrategy; import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.cluster.api.SimpleMap; import org.apache.solr.common.ConfigNode; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; @@ -1051,87 +1050,10 @@ public ConfigNode get(String name) { //there is no overlay return root.get(name); } - return new OverlaidConfigNode(name, null,root.get(name)); + return new OverlaidConfigNode(overlay, name, null,root.get(name)); } public ConfigNode get(String name, Predicate test) { return root.get(name, test); } - private class OverlaidConfigNode implements ConfigNode { - - private final String _name; - private final ConfigNode delegate; - private final OverlaidConfigNode parent; - - private OverlaidConfigNode(String name, OverlaidConfigNode parent, ConfigNode delegate) { - this._name = name; - this.delegate = delegate; - this.parent = parent; - } - - private List path(List path) { - if(path== null) path = new ArrayList<>(5); - try { - if (parent != null) return parent.path(path); - } finally { - path.add(_name); - } - return path; - } - - @Override - public ConfigNode get(String name) { - return wrap(delegate.get(name), name); - } - - private ConfigNode wrap(ConfigNode n, String name) { - return new OverlaidConfigNode(name,this, n); - } - - @Override - public ConfigNode get(String name, Predicate test) { - return wrap(delegate.get(name, test), name); - } - - @Override - public String txt() { - return overlayText(delegate.txt(), null); - } - - @Override - public ConfigNode get(String name, int idx) { - return wrap(delegate.get(name, idx), name); - } - - @Override - public String name() { - return delegate.name(); - } - @Override - public SimpleMap attributes() { - return delegate.attributes(); - } - - @Override - public boolean exists() { - return delegate.exists(); - } - - @Override - public String attr(String name) { - return overlayText(delegate.attr(name),name); - } - - private String overlayText(String def, String appendToPath) { - List path = path(null); - if(appendToPath !=null) path.add(appendToPath); - Object val = overlay.getXPathProperty(path); - return val ==null? def: val.toString(); - } - - @Override - public void forEachChild(Function fun) { - delegate.forEachChild(fun); - } - } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java index dc1e22fb3145..1fe7df21e26f 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/DOMUtil.java @@ -17,7 +17,9 @@ package org.apache.solr.common.util; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,7 +41,7 @@ public class DOMUtil { public static final String XML_RESERVED_PREFIX = "xml"; - public static final Set NL_TAGS = Set.of("str", "int","long","float","double","bool"); + public static final Set NL_TAGS = new HashSet<>(Arrays.asList("str", "int", "long", "float", "double", "bool", "null")); public static Map toMap(NamedNodeMap attrs) { From 934f6301092d56bd572f25fbbba594214e502c29 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 29 Apr 2021 23:22:07 +1000 Subject: [PATCH 3/3] CHANGES.txt --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ee93cf6b4104..08d7b829aa01 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -338,6 +338,8 @@ Improvements * SOLR-15155: Let CloudHttp2SolrClient accept an external Http2SolrClient Builder (Tomás Fernández Löbbe) +* SOLR-15337: Avoid XPath in solrconfig.xml parsing (noble) + Optimizations --------------------- * SOLR-15079: Block Collapse - Faster collapse code when groups are co-located via Block Join style nested doc indexing.