Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SOLR-13707: API to expose the currently used package name, details for each plugin #841

Merged
merged 3 commits into from
Aug 22, 2019
Merged

SOLR-13707: API to expose the currently used package name, details for each plugin #841

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8777e4f
Initial commit
noblepaul Aug 22, 2019
fbf223e
support /config/{componentType}/{componentName} format
noblepaul Aug 22, 2019
1911cd2
uncommented the AwaitsFix
noblepaul Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 9 additions & 12 deletions solr/core/src/java/org/apache/solr/core/PackageManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ public int getZNodeVersion(String pkg) {
Package p = pkgs.get(pkg);
return p == null ? -1 : p.lib.getZnodeVersion();
}
public RuntimeLib getLib(String name){
Package p = pkgs.get(name);
return p == null? null: p.lib;
}

static class Package implements MapWriter {
final RuntimeLib lib;
Expand All @@ -90,12 +94,8 @@ public String getName() {


public boolean isModified(Map map) {
if ((Objects.equals(lib.getSha512(), (map).get(SHA512)) &&
Objects.equals(lib.getSig(), (map).get(SHA512)))) {
return false;
} else {
return true;
}
return (!Objects.equals(lib.getSha512(), (map).get(SHA512)) ||
!Objects.equals(lib.getSig(), (map).get(SHA512)));
}
}

Expand Down Expand Up @@ -153,7 +153,7 @@ public boolean onChange(Map<String, Object> properties) {
private boolean updatePackages(Map<String, Object> properties, int ver) {
Map m = (Map) properties.getOrDefault(PACKAGE, Collections.emptyMap());
if (pkgs.isEmpty() && m.isEmpty()) return false;
boolean needsReload[] = new boolean[1];
boolean[] needsReload = new boolean[1];
if (m.size() == pkgs.size()) {
m.forEach((k, v) -> {
if (v instanceof Map) {
Expand Down Expand Up @@ -361,11 +361,8 @@ public void writeMap(EntryWriter ew) throws IOException {

public boolean shouldReload(Map metaData, Map<String, Package> pkgs) {
Package p = pkgs.get(pkg);
if (meta.equals(metaData) && p != null && p.lib.getZnodeVersion() <= zkversion) {
//the metadata is same and the package has not changed since we last loaded
return false;
}
return true;
//the metadata is same and the package has not changed since we last loaded
return !meta.equals(metaData) || p == null || p.lib.getZnodeVersion() > zkversion;
}
}
}
Expand Down
33 changes: 27 additions & 6 deletions solr/core/src/java/org/apache/solr/core/PluginBag.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.StrUtils;
Expand Down Expand Up @@ -121,7 +122,7 @@ private static <T> T createInitInstance(PluginInfo pluginInfo, SolrConfig.SolrPl
boolean isRuntimeLib) {
T localInst = null;
try {
localInst = (T) core.createInstance(pluginInfo.className, pluginMeta.clazz, pluginMeta.getCleanTag(), core, resourceLoader);
localInst = (T) SolrCore.createInstance(pluginInfo.className, pluginMeta.clazz, pluginMeta.getCleanTag(), core, resourceLoader);
} catch (SolrException e) {
if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) {
throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()),
Expand Down Expand Up @@ -359,7 +360,7 @@ public PluginHolder<T> createPlugin(PluginInfo info) {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
T inst = SolrCore.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
initInstance(inst, info);
return new PluginHolder<>(info, inst);
}
Expand Down Expand Up @@ -490,15 +491,34 @@ public class PackagePluginHolder<T> extends PluginHolder<T> {
private final SolrConfig.SolrPluginInfo pluginMeta;
private final PackageManager packageManager;
private final String pkg;
private int znodeVersion =-1;
private RuntimeLib runtimeLib;

public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
this.core = core;
this.pluginMeta = pluginMeta;
this.pkg = info.attributes.get(CommonParams.PACKAGE);
this.core.addPackageListener(pkg, (lib) -> {
if(lib.getZnodeVersion() > znodeVersion) reload();
this.core.addPackageListener(new SolrCore.PkgListener() {
@Override
public String packageName() {
return pkg;
}

@Override
public PluginInfo pluginInfo() {
return info;
}

@Override
public MapWriter lib() {
return runtimeLib;
}

@Override
public void changed(RuntimeLib lib) {
int myVersion = runtimeLib == null? -1 : runtimeLib.znodeVersion;
if(lib.getZnodeVersion() > myVersion) reload();
}
});
this.packageManager = core.getCoreContainer().getPackageManager();
reload();
Expand All @@ -509,7 +529,8 @@ private void reload() {
if(inst == null) log.info("reloading plugin {} ", pluginInfo.name);
inst = createInitInstance(pluginInfo, pluginMeta,
core, packageManager.getResourceLoader(this.pkg), true);
znodeVersion = packageManager.getZNodeVersion(pkg);
this.runtimeLib = packageManager.getLib(pkg);

}


Expand Down
11 changes: 7 additions & 4 deletions solr/core/src/java/org/apache/solr/core/PluginInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,15 @@ public class PluginInfo implements MapSerializable {
public final List<PluginInfo> children;
private boolean isFromSolrConfig;

public List<String> pathInConfig;

public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
this.type = type;
this.name = attrs.get(NAME);
this.className = attrs.get(CLASS_NAME);
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList() : unmodifiableList(children);
this.children = children == null ? Collections.emptyList() : unmodifiableList(children);
isFromSolrConfig = false;
}

Expand Down Expand Up @@ -97,7 +99,7 @@ public PluginInfo(String type, Map<String, Object> map) {
this.name = (String) m.get(NAME);
this.className = (String) m.get(CLASS_NAME);
attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList();
this.children = Collections.emptyList();
isFromSolrConfig = true;
}

Expand All @@ -112,7 +114,7 @@ private List<PluginInfo> loadSubPlugins(Node node) {
PluginInfo pluginInfo = new PluginInfo(nd, null, false, false);
if (pluginInfo.isEnabled()) children.add(pluginInfo);
}
return children.isEmpty() ? Collections.<PluginInfo>emptyList() : unmodifiableList(children);
return children.isEmpty() ? Collections.emptyList() : unmodifiableList(children);
}

@Override
Expand Down Expand Up @@ -178,7 +180,7 @@ public List<PluginInfo> getChildren(String type) {
return result;
}

public static final PluginInfo EMPTY_INFO = new PluginInfo("", Collections.<String, String>emptyMap(), new NamedList(), Collections.<PluginInfo>emptyList());
public static final PluginInfo EMPTY_INFO = new PluginInfo("", Collections.emptyMap(), new NamedList(), Collections.emptyList());

private static final HashSet<String> NL_TAGS = new HashSet<>
(asList("lst", "arr",
Expand All @@ -199,6 +201,7 @@ public PluginInfo copy() {
PluginInfo result = new PluginInfo(type, attributes,
initArgs != null ? initArgs.clone() : null, children);
result.isFromSolrConfig = isFromSolrConfig;
result.pathInConfig = pathInConfig;
return result;
}

Expand Down
49 changes: 29 additions & 20 deletions solr/core/src/java/org/apache/solr/core/SolrConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
Expand All @@ -55,6 +56,7 @@
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
Expand Down Expand Up @@ -271,7 +273,7 @@ public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
args.put("initialSize", "10");
args.put("showItems", "-1");
args.put("class", FastLRUCache.class.getName());
conf = new CacheConfig(args);
conf = new CacheConfig(args,"query/fieldValueCache");
}
fieldValueCacheConfig = conf;
useColdSearcher = getBool("query/useColdSearcher", false);
Expand All @@ -298,7 +300,7 @@ public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
List<PluginInfo> caches = getPluginInfos(SolrCache.class.getName());
if (!caches.isEmpty()) {
for (PluginInfo c : caches) {
userCacheConfigs.put(c.name, new CacheConfig(c.attributes));
userCacheConfigs.put(c.name, new CacheConfig(c.attributes, StrUtils.join(c.pathInConfig, '/')));
}
}
this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs);
Expand Down Expand Up @@ -372,17 +374,17 @@ public static final Version parseLuceneVersionString(final String matchVersion)
.add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
// TODO: WTF is up with queryConverter???
// it apparently *only* works as a singleton? - SOLR-4304
// and even then -- only if there is a single SpellCheckComponent
// because of queryConverter.setIndexAnalyzer
.add(new SolrPluginInfo(SolrCache.class, SolrCache.TYPE, REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
// TODO: WTF is up with queryConverter???
// it apparently *only* works as a singleton? - SOLR-4304
// and even then -- only if there is a single SpellCheckComponent
// because of queryConverter.setIndexAnalyzer
.add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS))
.add(new SolrPluginInfo(RuntimeLib.class, RuntimeLib.TYPE, REQUIRE_NAME, MULTI_OK))
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
// way around that in the PluginInfo framework
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
// way around that in the PluginInfo framework
.add(new SolrPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK, REQUIRE_NAME_IN_OVERLAY))
.add(new SolrPluginInfo(SolrEventListener.class, "//listener", REQUIRE_CLASS, MULTI_OK, REQUIRE_NAME_IN_OVERLAY))

Expand Down Expand Up @@ -532,6 +534,9 @@ public List<PluginInfo> readPluginInfos(String tag, boolean requireName, boolean
NodeList nodes = (NodeList) evaluate(tag, XPathConstants.NODESET);
for (int i = 0; i < nodes.getLength(); i++) {
PluginInfo pluginInfo = new PluginInfo(nodes.item(i), "[solrconfig.xml] " + tag, requireName, requireClass);
if (requireName) {
pluginInfo.pathInConfig = Arrays.asList(tag, pluginInfo.name);
}
if (pluginInfo.isEnabled()) result.add(pluginInfo);
}
return result;
Expand Down Expand Up @@ -605,7 +610,7 @@ public Map<String, Object> toMap(Map<String, Object> map) {
"cacheControl", cacheControlHeader);
}

public static enum LastModFrom {
public enum LastModFrom {
OPENTIME, DIRLASTMOD, BOGUS;

/**
Expand Down Expand Up @@ -757,20 +762,24 @@ public List<PluginInfo> getPluginInfos(String type) {
Map<String, Map> infos = overlay.getNamedPlugins(info.getCleanTag());
if (!infos.isEmpty()) {
LinkedHashMap<String, PluginInfo> map = new LinkedHashMap<>();
if (result != null) for (PluginInfo pluginInfo : result) {
//just create a UUID for the time being so that map key is not null
String name = pluginInfo.name == null ?
UUID.randomUUID().toString().toLowerCase(Locale.ROOT) :
pluginInfo.name;
map.put(name, pluginInfo);
if (result != null) {
for (PluginInfo pluginInfo : result) {
//just create a UUID for the time being so that map key is not null
String name = pluginInfo.name == null ?
UUID.randomUUID().toString().toLowerCase(Locale.ROOT) :
pluginInfo.name;
map.put(name, pluginInfo);
}
}
for (Map.Entry<String, Map> e : infos.entrySet()) {
map.put(e.getKey(), new PluginInfo(info.getCleanTag(), e.getValue()));
PluginInfo value = new PluginInfo(info.getCleanTag(), e.getValue());
value.pathInConfig = Arrays.asList(info.getCleanTag(),e.getKey());
map.put(e.getKey(), value);
}
result = new ArrayList<>(map.values());
}
}
return result == null ? Collections.<PluginInfo>emptyList() : result;
return result == null ? Collections.emptyList() : result;
}

public PluginInfo getPluginInfo(String type) {
Expand Down
34 changes: 23 additions & 11 deletions solr/core/src/java/org/apache/solr/core/SolrCore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;

import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
Expand All @@ -80,6 +79,7 @@
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
Expand All @@ -95,7 +95,6 @@
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.DirectoryFactory.DirContext;
Expand Down Expand Up @@ -240,13 +239,17 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public volatile boolean searchEnabled = true;
public volatile boolean indexEnabled = true;
public volatile boolean readOnly = false;
private List<Pair<String ,Consumer<RuntimeLib>>> packageListeners = new ArrayList<>();
private List<PkgListener> packageListeners = new ArrayList<>();


public Set<String> getMetricNames() {
return metricNames;
}

public List<PkgListener> getPackageListeners(){
return Collections.unmodifiableList(packageListeners);
}

public Date getStartTimeStamp() {
return startTime;
}
Expand Down Expand Up @@ -358,12 +361,23 @@ public String getIndexDir() {
}

void packageUpdated(RuntimeLib lib) {
for (Pair<String, Consumer<RuntimeLib>> pair : packageListeners) {
if(lib.equals(pair.first())) pair.second().accept(lib);
for (PkgListener listener : packageListeners) {
if(lib.getName().equals(listener.packageName())) listener.changed(lib);
}
}
public void addPackageListener(String pkg, Consumer<RuntimeLib> r){
packageListeners.add(new Pair<>(pkg, r));
public void addPackageListener(PkgListener listener){
packageListeners.add(listener);
}

public interface PkgListener {

String packageName();

PluginInfo pluginInfo();

void changed(RuntimeLib lib);

MapWriter lib();
}


Expand Down Expand Up @@ -851,7 +865,7 @@ private UpdateHandler createReloadedUpdateHandler(String className, String msg,
for (Constructor<?> con : cons) {
Class<?>[] types = con.getParameterTypes();
if (types.length == 2 && types[0] == SolrCore.class && types[1] == UpdateHandler.class) {
return UpdateHandler.class.cast(con.newInstance(this, updateHandler));
return (UpdateHandler) con.newInstance(this, updateHandler);
}
}
throw new SolrException(ErrorCode.SERVER_ERROR, "Error Instantiating " + msg + ", " + className + " could not find proper constructor for " + UpdateHandler.class.getName());
Expand Down Expand Up @@ -2423,7 +2437,6 @@ public RefCounted<SolrIndexSearcher> getSearcher(boolean forceNew, boolean retur

if (!success) {
newSearcherOtherErrorsCounter.inc();
;
synchronized (searcherLock) {
onDeckSearchers--;

Expand Down Expand Up @@ -3126,8 +3139,7 @@ private static boolean checkStale(SolrZkClient zkClient, String zkPath, int curr
try {
Stat stat = zkClient.exists(zkPath, null, true);
if (stat == null) {
if (currentVersion > -1) return true;
return false;
return currentVersion > -1;
}
if (stat.getVersion() > currentVersion) {
log.debug("{} is stale will need an update from {} to {}", zkPath, currentVersion, stat.getVersion());
Expand Down
Loading