Skip to content
Permalink
Browse files

Small perf improvement : initialize threads names early when possible

Initializing Thread names using the Thread constructor parameter is
faster as it already sets a thread name even if no customized one is
given, while an additional call to the Thread.setName() function
internally do synchronized access, eventually runs access check on the
security manager and performs a native call.

Profiling a running YaCy server revealed that the total processing time
spent on Thread.setName() for a typical p2p search was in the range of
seconds.
  • Loading branch information...
luccioman committed May 23, 2018
1 parent 79bd9f6 commit fa4399d5d2c11ee08f7f50eab4cd014b1d9a491b
Showing with 40 additions and 66 deletions.
  1. +1 −2 source/net/yacy/cora/federate/opensearch/SRURSSConnector.java
  2. +1 −2 source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
  3. +2 −4 source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
  4. +2 −4 source/net/yacy/cora/federate/solr/instance/ServerShard.java
  5. +1 −2 source/net/yacy/cora/protocol/Domains.java
  6. +1 −2 source/net/yacy/cora/protocol/ftp/FTPClient.java
  7. +1 −2 source/net/yacy/cora/protocol/http/HTTPClient.java
  8. +5 −1 source/net/yacy/cora/sorting/Array.java
  9. +1 −2 source/net/yacy/cora/storage/Files.java
  10. +1 −2 source/net/yacy/crawler/CrawlStacker.java
  11. +1 −2 source/net/yacy/crawler/data/Cache.java
  12. +1 −2 source/net/yacy/crawler/data/Transactions.java
  13. +1 −2 source/net/yacy/crawler/robots/RobotsTxt.java
  14. +1 −2 source/net/yacy/document/content/dao/PhpBB3Dao.java
  15. +1 −2 source/net/yacy/document/parser/pdfParser.java
  16. +1 −2 source/net/yacy/gui/YaCyApp.java
  17. +1 −2 source/net/yacy/http/Jetty9HttpServerImpl.java
  18. +1 −2 source/net/yacy/kelondro/blob/MapHeap.java
  19. +1 −1 source/net/yacy/kelondro/rwi/IODispatcher.java
  20. +2 −1 source/net/yacy/kelondro/rwi/IndexCell.java
  21. +1 −2 source/net/yacy/kelondro/table/SplitTable.java
  22. +2 −4 source/net/yacy/peers/RemoteSearch.java
  23. +1 −1 source/net/yacy/peers/graphics/WebStructureGraph.java
  24. +1 −1 source/net/yacy/search/MemoryTracker.java
  25. +6 −12 source/net/yacy/search/Switchboard.java
  26. +1 −3 source/net/yacy/search/query/SearchEvent.java
  27. +1 −2 source/net/yacy/search/query/SecondarySearchSuperviser.java
@@ -49,10 +49,9 @@ public static Thread searchSRURSS(
final CacheStrategy verify,
final boolean global,
final ClientIdentification.Agent agent) {
final Thread job = new Thread() {
final Thread job = new Thread("searchSRURSS:" + urlBase) {
@Override
public void run() {
Thread.currentThread().setName("searchSRURSS:" + urlBase);
int startRecord = 0;
RSSMessage message;
int maximumRecords = maximumRecordsInit;
@@ -206,10 +206,9 @@ protected static LoadTimeURL getLoadTimeURL(final Object doc) {
final long endtime = maxtime < 0 || maxtime == Long.MAX_VALUE ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime; // we know infinity!
final Thread[] t = new Thread[concurrency];
for (int i = 0; i < Math.max(1, concurrency); i++) {
t[i] = new Thread() {
t[i] = new Thread("AbstractSolrConnector:concurrentDocumentsByQueriesWithPrefetch(" + querystrings.size() + " queries, first: " + querystrings.iterator().next() + ")") {
@Override
public void run() {
this.setName("AbstractSolrConnector:concurrentDocumentsByQueriesWithPrefetch(" + querystrings.size() + " queries, first: " + querystrings.iterator().next() + ")");
String nextID;
try {
while (System.currentTimeMillis() < endtime && (nextID = idQueue.take()) != AbstractSolrConnector.POISON_ID) {
@@ -357,20 +357,18 @@ public long getCountByQuery(final String querystring) throws IOException {
return this.solr1.getCountByQuery(querystring);
}
final AtomicLong count = new AtomicLong(0);
Thread t0 = new Thread() {
Thread t0 = new Thread("MirrorSolrConnector.getCountByQuery/t0") {
@Override
public void run() {
this.setName("MirrorSolrConnector.getCountByQuery/t0");
try {
count.addAndGet(MirrorSolrConnector.this.solr0.getCountByQuery(querystring));
} catch (final IOException e) {}
}
};
t0.start();
Thread t1 = new Thread() {
Thread t1 = new Thread("MirrorSolrConnector.getCountByQuery/t1") {
@Override
public void run() {
this.setName("MirrorSolrConnector.getCountByQuery/t1");
try {
count.addAndGet(MirrorSolrConnector.this.solr1.getCountByQuery(querystring));
} catch (final IOException e) {}
@@ -388,10 +388,9 @@ public QueryResponse query(final SolrParams params) throws SolrServerException,
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
List<Thread> t = new ArrayList<Thread>();
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
Thread t0 = new Thread("ServerShard.query/1(" + params.toString() + ")") {
@Override
public void run() {
this.setName("ServerShard.query/1(" + params.toString() + ")");
QueryResponse rsp;
try {
rsp = s.query(params);
@@ -427,10 +426,9 @@ public QueryResponse query(final SolrParams params, final METHOD method) throws
// concurrently call all shards
List<Thread> t = new ArrayList<Thread>();
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
Thread t0 = new Thread("ServerShard.query/2(" + params.toString() + ")") {
@Override
public void run() {
this.setName("ServerShard.query/2(" + params.toString() + ")");
QueryResponse rsp;
try {
rsp = s.query(params, method);
@@ -121,10 +121,9 @@
// if such a lookup blocks, it can cause that the static initiatializer does not finish fast
// therefore we start the host name lookup as concurrent thread
// meanwhile the host name is "127.0.0.1" which is not completely wrong
new Thread() {
new Thread("Domains: init") {
@Override
public void run() {
Thread.currentThread().setName("Domains: init");
// try to get local addresses from interfaces
try {
final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
@@ -2556,11 +2556,10 @@ private static void dir(final String host, final String remotePath, final String
ftpClient.open(host, port);
ftpClient.login(user, pw);
final LinkedBlockingQueue<entryInfo> queue = new LinkedBlockingQueue<entryInfo>();
new Thread() {
new Thread("FTP.sitelist(" + host + ":" + port + ")") {
@Override
public void run() {
try {
Thread.currentThread().setName("FTP.sitelist(" + host + ":" + port + ")");
sitelist(ftpClient, path, queue, depth);
ftpClient.quit();
} catch (final Exception e) {} finally {
@@ -1122,8 +1122,7 @@ public static void main(final String[] args) {
private volatile boolean shutdown;

public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
super();
this.setName("HTTPClient.IdleConnectionMonitorThread");
super("HTTPClient.IdleConnectionMonitorThread");
this.connMgr = connMgr;
}

@@ -63,9 +63,13 @@ public static void terminate() {
}

private static class SortJobWorker extends Thread {

public SortJobWorker() {
super("Array.SortJobWorker");
}

@Override
public void run() {
this.setName("Array.SortJobWorker");
SortJob<?> job;
try {
while ((job = sortJobs.take()) != POISON_JOB_WORKER) {
@@ -80,10 +80,9 @@ public static InputStream read(File f) throws IOException {
final BlockingQueue<String> q = new LinkedBlockingQueue<String>();
final InputStream is = read(f);
final BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
Thread t = new Thread() {
Thread t = new Thread("Files.concurrentLineReader:" + f) {
@Override
public void run() {
Thread.currentThread().setName("Files.concurrentLineReader:" + f);
String line;
try {
while ((line = br.readLine()) != null) {
@@ -271,10 +271,9 @@ public void enqueueEntriesFTP(
final String host = ftpURL.getHost();
final int port = ftpURL.getPort();
final int pathParts = ftpURL.getPaths().length;
new Thread() {
new Thread("enqueueEntriesFTP") {
@Override
public void run() {
Thread.currentThread().setName("enqueueEntriesFTP");
BlockingQueue<FTPClient.entryInfo> queue;
try {
queue = FTPClient.sitelist(host, port, user, pw, ftpURL.getPath(), profile.depth());
@@ -148,10 +148,9 @@ public static void init(final File htCachePath, final String peerSalt, final lon
// We do this as a concurrent job only once after start-up silently
if (responseHeaderDB.size() != fileDB.size()) {
ConcurrentLog.warn("Cache", "file and metadata size is not equal, starting a cleanup thread...");
Thread startupCleanup = new Thread() {
Thread startupCleanup = new Thread("Cache startupCleanup") {
@Override
public void run() {
Thread.currentThread().setName("Cache startupCleanup");
// enumerate the responseHeaderDB and find out all entries that are not inside the fileDBunbuffered
BlockingQueue<byte[]> q = responseHeaderDB.keyQueue(1000);
final HandleSet delkeys = new RowHandleSet(Word.commonHashLength, Base64Order.enhancedCoder, 1);
@@ -211,10 +211,9 @@ public static boolean store(final DigestURL url, final Date date, final int dept
final String urls = url.toNormalform(true);
final File pdfPath = Transactions.definePath(url, depth, date, "pdf", Transactions.State.INVENTORY);
if (concurrency && executorRunning.intValue() < Runtime.getRuntime().availableProcessors()) {
Thread t = new Thread(){
Thread t = new Thread("Transactions.store"){
@Override
public void run() {
this.setName("Transactions.store");
executorRunning.incrementAndGet();
try {
Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent, acceptLanguage, pdfPath);
@@ -235,10 +235,9 @@ public void ensureExist(final MultiProtocolURL theURL, final ClientIdentificatio
return;
}
if (robotsTable != null && robotsTable.containsKey(robotsTable.encodedKey(urlHostPort))) return;
Thread t = new Thread() {
Thread t = new Thread("Robots.txt:ensureExist(" + theURL.toNormalform(true) + ")") {
@Override
public void run(){
this.setName("Robots.txt:ensureExist(" + theURL.toNormalform(true) + ")");
// make or get a synchronization object
DomSync syncObj = RobotsTxt.this.syncObjects.get(urlHostPort);
if (syncObj == null) {
@@ -177,10 +177,9 @@ private DCEntry getOne(String sql) {
private BlockingQueue<DCEntry> toQueue(final StringBuilder sql, int queueSize) {
// execute the query and push entries to a queue concurrently
final BlockingQueue<DCEntry> queue = new ArrayBlockingQueue<DCEntry>(queueSize);
Thread dbreader = new Thread() {
Thread dbreader = new Thread("PhpBB3Dao.toQueue") {
@Override
public void run() {
Thread.currentThread().setName("PhpBB3Dao.toQueue");
Statement stmt = null;
ResultSet rs = null;
try {
@@ -206,10 +206,9 @@ public pdfParser() {
stripper.setEndPage(Integer.MAX_VALUE); // set to default
// we start the pdf parsing in a separate thread to ensure that it can be terminated
final PDDocument pdfDocC = pdfDoc;
final Thread t = new Thread() {
final Thread t = new Thread("pdfParser.getText:" + location) {
@Override
public void run() {
Thread.currentThread().setName("pdfParser.getText:" + location);
try {
writer.append(stripper.getText(pdfDocC));
} catch (final Throwable e) {}
@@ -164,10 +164,9 @@ public void actionPerformed(ActionEvent e) {

// registering shutdown hook
log.info("Registering Shutdown Hook");
Thread t = new Thread() {
Thread t = new Thread("YaCyApp") {
@Override
public void run() {
Thread.currentThread().setName("YaCyApp");
app = new Application("YaCy GUI", operation, menues, new InfoPage(host, port));
app.setLocationRelativeTo(null);
app.setVisible(true);
@@ -298,11 +298,10 @@ public int getSslPort() {
@Override
public void reconnect(final int milsec) {

new Thread() {
new Thread("Jetty8HttpServer.reconnect") {

@Override
public void run() {
this.setName("Jetty8HttpServer.reconnect");
try {
Thread.sleep(milsec);
} catch (final InterruptedException e) {
@@ -526,10 +526,9 @@ public void putAll(final Map<? extends byte[], ? extends Map<String, String>> ma
public final static byte[] POISON_QUEUE_ENTRY = "POISON".getBytes();
public BlockingQueue<byte[]> keyQueue(final int size) {
final ArrayBlockingQueue<byte[]> set = new ArrayBlockingQueue<byte[]>(size);
(new Thread() {
(new Thread("MapHeap.keyQueue:" + size) {
@Override
public void run() {
Thread.currentThread().setName("MapHeap.keyQueue:" + size);
try {
final Iterator<byte[]> i = MapHeap.this.blob.keys(true, false);
while (i.hasNext())
@@ -58,13 +58,13 @@
private final int writeBufferSize;

public IODispatcher(final int dumpQueueLength, final int mergeQueueLength, final int writeBufferSize) {
super("IODispatcher");
this.termination = new Semaphore(0);
this.controlQueue = new Semaphore(0);
this.dumpQueue = new ArrayBlockingQueue<DumpJob<? extends Reference>>(dumpQueueLength);
this.mergeQueue = new ArrayBlockingQueue<MergeJob>(mergeQueueLength);
this.writeBufferSize = writeBufferSize;
this.terminate = false;
this.setName("IODispatcher");
}

public void terminate() {
@@ -113,8 +113,9 @@ public IndexCell(

private class FlushThread extends Thread {
public FlushThread(String name) {
this.setName("IndexCell.FlushThread(" + name + ")");
super("IndexCell.FlushThread(" + name + ")");
}

@Override
public void run() {
while (IndexCell.this.flushShallRun) {
@@ -231,10 +231,9 @@ private void init() {
}
}
final Table a = table;
final Thread p = new Thread() {
final Thread p = new Thread("SplitTable.warmUp") {
@Override
public void run() {
Thread.currentThread().setName("SplitTable.warmUp");
a.warmUp();
}
};
@@ -331,10 +331,9 @@ public static Thread secondaryRemoteSearch(
// prepare seed targets and threads
final Seed targetPeer = event.peers.getConnected(targethash);
if (targetPeer == null || targetPeer.hash == null) return null;
Thread secondary = new Thread() {
Thread secondary = new Thread("RemoteSearch.secondaryRemoteSearch(" + wordhashes + " to " + targethash + ")") {
@Override
public void run() {
this.setName("RemoteSearch.secondaryRemoteSearch(" + wordhashes + " to " + targethash + ")");
event.oneFeederStarted();
try {
int urls = Protocol.secondarySearch(
@@ -400,10 +399,9 @@ public static Thread solrRemoteSearch(
// check own peer status
if (event.peers.mySeed() == null) { return null; }
// prepare threads
Thread solr = new Thread() {
Thread solr = new Thread("RemoteSearch.solrRemoteSearch(" + solrQuery.getQuery() + " to " + (targetPeer == null ? "myself" : targetPeer.hash) + ")") {
@Override
public void run() {
this.setName("RemoteSearch.solrRemoteSearch(" + solrQuery.getQuery() + " to " + (targetPeer == null ? "myself" : targetPeer.hash) + ")");
int urls = 0;
try {
event.oneFeederStarted();
@@ -180,7 +180,7 @@ public WebStructureGraph(final File structureFile) {
*/
private class PublicRefDNSResolvingProcess extends Thread {
private PublicRefDNSResolvingProcess() {
this.setName("WebStructureGraph.PublicRefDNSResolvingProcess");
super("WebStructureGraph.PublicRefDNSResolvingProcess");
}

@Override
@@ -46,9 +46,9 @@ public static void stopSystemProfiling() {
private boolean running;

public MemoryTracker(final long time) {
super("MemoryTracker");
this.delaytime = time;
running = true;
this.setName("MemoryTracker");
}

@Override
@@ -416,10 +416,9 @@ public boolean jobImpl() throws Exception {

// init libraries
this.log.config("initializing libraries");
new Thread() {
new Thread("LibraryProvider.initialize") {
@Override
public void run() {
Thread.currentThread().setName("LibraryProvider.initialize");
LibraryProvider.initialize(Switchboard.this.dictionariesPath);
// persistent Vocabulary Switches
final Set<String> omit = Switchboard.this.getConfigSet("search.result.show.vocabulary.omit");
@@ -857,10 +856,9 @@ public boolean jobImpl() throws Exception {

// init bookmarks DB: needs more time since this does a DNS lookup for each Bookmark.
// Can be started concurrently
new Thread() {
new Thread("Switchboard.initBookmarks") {
@Override
public void run() {
Thread.currentThread().setName("Switchboard.initBookmarks");
try {
initBookmarks();
} catch (final IOException e ) {
@@ -1281,10 +1279,9 @@ public void setHttpServer(YaCyHttpServer server) {
super.setHttpServer(server);

// finally start jobs which shall be started after start-up
new Thread() {
new Thread("Switchboard.setHttpServer") {
@Override
public void run() {
Thread.currentThread().setName("Switchboard.setHttpServer");
try {Thread.sleep(10000);} catch (final InterruptedException e) {} // needs httpd up
schedulerJob(); // trigger startup actions
}
@@ -3450,10 +3447,9 @@ public void stackURLs(Set<DigestURL> rootURLs, final CrawlProfile profile, final
final List<Thread> stackthreads = new ArrayList<Thread>(); // do this concurrently
for (DigestURL url: rootURLs) {
final DigestURL turl = url;
Thread t = new Thread() {
Thread t = new Thread("Switchboard.stackURLs") {
@Override
public void run() {
this.setName("Switchboard.stackURLs");
String failreason;
if ((failreason = Switchboard.this.stackUrl(profile, turl)) == null) successurls.add(turl); else failurls.put(turl, failreason);
}
@@ -4018,10 +4014,9 @@ public boolean dhtTransferJob() {
}

public final void heuristicSite(final SearchEvent searchEvent, final String host) {
new Thread() {
new Thread("Switchboard.heuristicSite:" + host) {
@Override
public void run() {
Thread.currentThread().setName("Switchboard.heuristicSite:" + host);
String r = host;
if ( r.indexOf("//", 0) < 0 ) {
r = "http://" + r;
@@ -4132,10 +4127,9 @@ public final void heuristicRSS(
final SearchEvent searchEvent,
final String feedName) {

new Thread() {
new Thread("heuristicRSS:" + feedName) {
@Override
public void run() {
Thread.currentThread().setName("heuristicRSS:" + feedName);
final DigestURL url;
try {
url = new DigestURL(MultiProtocolURL.unescape(urlpattern));

0 comments on commit fa4399d

Please sign in to comment.
You can’t perform that action at this time.