Skip to content
Permalink
Browse files

Added and updated hint messages about remote crawler status

To help identify why remote crawl results may not be received.
  • Loading branch information...
luccioman committed Jul 6, 2018
1 parent c726154 commit 534f09e92b2ae7984c401f4f8d8b2921dd97b6cd
@@ -28,9 +28,11 @@ <h2>(1) Results of Remote Crawl Receipts</h2>
but had been crawled by <em>other</em> peers.
This is the 'mirror'-case of process (6).
</p>
<p><em>Use Case:</em> You get entries here, if you start a local crawl on the 'Index Creation'-Page and check the
'Do Remote Indexing'-flag. Every page that a remote peer indexes upon this peer's request
is reported back and can be monitored here.</p>
<p><em>Use Case:</em> You get entries here, if you start a local crawl on the '<a href="CrawlStartExpert.html">Advanced Crawler</a>' page and check the
'Do Remote Indexing'-flag, and if you checked the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page.
</p>
<p>Every page that a remote peer indexes upon this peer's request is reported back and can be monitored here.</p>
#(remoteCrawlerDisabled)#::<div class="info"><p>No remote crawl results can currently been added to the local index as the remote crawler is disabled on this peer.<p></div>#(/remoteCrawlerDisabled)#
::
<h2>(2) Results for Result of Search Queries</h2>
<p>This index transfer was initiated by your peer by doing a search query.
@@ -59,7 +61,8 @@ <h2>(5) Results for Local Crawling</h2>
<h2>(6) Results for Global Crawling</h2>
<p>These pages had been indexed by your peer, but the crawl was initiated by a remote peer.
This is the 'mirror'-case of process (1).</p>
<p><em>Use Case:</em> This list may fill if you check the 'Accept remote crawling requests'-flag on the '<a href="RemoteCrawl_p.html">Index Create</a>' page</p>
<p><em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page</p>
#(remoteCrawlerDisabled)#::<div class="info"><p>The remote crawler is currently disabled<p></div>#(/remoteCrawlerDisabled)#
::
<h2>(7) Results from surrogates import</h2>
<p>These records had been imported from surrogate files in DATA/SURROGATES/in</p>
@@ -50,6 +50,7 @@
import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -348,6 +349,7 @@ public static serverObjects respond(final RequestHeader header, serverObjects po
}

prop.put("process", tabletype.getCode());
prop.put("process_remoteCrawlerDisabled", !sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
// return rewrite properties
return prop;
}
@@ -98,10 +98,17 @@
}
// Remote crawl
var remoteCrawlerDisabledInfo = document.getElementById('remoteCrawlerDisabledInfo');
if ($('#crawlOrder').isChecked()) {
if(remoteCrawlerDisabledInfo != null) {
remoteCrawlerDisabledInfo.className = '';
}
$('#intention').enable();
if (cId === "crawlOrder") { $('#intention').focus(); }
} else {
if(remoteCrawlerDisabledInfo != null) {
remoteCrawlerDisabledInfo.className = 'hidden';
}
$('#intention').disable();
}
}
@@ -623,9 +630,16 @@ <h2>Expert Crawl Start</h2>
so they can omit starting a crawl with the same start point.
</span></span>
<table style="border-width: 0px">
#(remoteCrawlerDisabled)#::
<tr #(crawlOrderChecked)#class="hidden"::#(/crawlOrderChecked)# id="remoteCrawlerDisabledInfo">
<td colspan="2"><div class="info"><p>Remote crawl results won't be added to the local index as the remote crawler is disabled on this peer.</p>
<p>You can activate it in the <a href="RemoteCrawl_p.html">Remote Crawl Configuration</a> page.</p></div>
</td>
</tr>
#(/remoteCrawlerDisabled)#
<tr>
<td>
<input type="checkbox" name="crawlOrder" id="crawlOrder" #(crawlOrderChecked)#::checked="checked"#(/crawlOrderChecked)# />
<input type="checkbox" name="crawlOrder" id="crawlOrder" #(crawlOrderChecked)#::checked="checked"#(/crawlOrderChecked)#/>
</td>
<td>
<label for="intention">Describe your intention to start this global crawl (optional)</label>:<br />
@@ -620,7 +620,10 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
// Do Remote Indexing?
if (sb.isP2PMode()) {
prop.put("remoteindexing", 1);
prop.put("remoteindexing_crawlOrderChecked", env.getConfigBool("crawlOrder", true) ? 1 : 0);
prop.put("remoteindexing_remoteCrawlerDisabled",
!sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
prop.put("remoteindexing_remoteCrawlerDisabled_crawlOrderChecked", env.getConfigBool("crawlOrder", true));
prop.put("remoteindexing_crawlOrderChecked", env.getConfigBool("crawlOrder", true));
prop.put("remoteindexing_intention", "");
} else {
prop.put("remoteindexing", 0);
@@ -632,7 +635,10 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
post.getBoolean("indexMedia") ? 1 : 0);
if (sb.isP2PMode()) {
prop.put("remoteindexing", 1);
prop.put("remoteindexing_crawlOrderChecked", post.getBoolean("crawlOrder") ? 1 : 0);
prop.put("remoteindexing_remoteCrawlerDisabled",
!sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
prop.put("remoteindexing_remoteCrawlerDisabled_crawlOrderChecked", post.getBoolean("crawlOrder"));
prop.put("remoteindexing_crawlOrderChecked", post.getBoolean("crawlOrder"));
prop.put("remoteindexing_intention", post.get("intention", ""));
} else {
prop.put("remoteindexing", 0);
@@ -227,6 +227,13 @@ <h2>Crawler</h2>
#(/info)#
</p>

#(wontReceiptRemoteResults)#::
<div class="alert alert-warning">
<p>You asked for remote indexing, but remote crawl results won't be added to the local index as the remote crawler is currently disabled on this peer.</p>
<p>You can activate it in the <a href="RemoteCrawl_p.html">Remote Crawl Configuration</a> page.</p>
</div>
#(/wontReceiptRemoteResults)#

<!-- #(noEmbeddedSolr)#::<div class="alert alert-error">No embedded local Solr index is connected. This is required to use the Solr filter query.
You can configure this with the <a href="IndexFederated_p.html">Index Sources &amp; targets</a> page.</div>
#(/noEmbeddedSolr)#
@@ -651,6 +651,9 @@ public static serverObjects respond(final RequestHeader header, final serverObje

// start the crawl
if(hasCrawlstartDataOK) {

final boolean wontReceiptRemoteRsults = crawlOrder && !sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false);

if ("url".equals(crawlingMode)) {
// stack requests
sb.crawler.putActive(handle, profile);
@@ -694,14 +697,19 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.putHTML("info_crawlingURL", (post.get("crawlingURL")));
prop.putHTML("info_reasonString", fr.toString());
}
if (successurls.size() > 0) sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
if (successurls.size() > 0) {
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);

prop.put("wontReceiptRemoteResults", wontReceiptRemoteRsults);
}
} else if ("sitemap".equals(crawlingMode)) {
try {
final DigestURL sitemapURL = sitemapURLStr.indexOf("//") > 0 ? new DigestURL(sitemapURLStr) : new DigestURL(rootURLs.iterator().next(), sitemapURLStr); // fix for relative paths which should not exist but are used anyway
sb.crawler.putActive(handle, profile);
final SitemapImporter importer = new SitemapImporter(sb, sitemapURL, profile);
importer.start();
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
prop.put("wontReceiptRemoteResults", wontReceiptRemoteRsults);
} catch (final Exception e) {
// mist
prop.put("info", "6");//Error with url
@@ -740,6 +748,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
ConcurrentLog.logException(e);
}
sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
prop.put("wontReceiptRemoteResults", wontReceiptRemoteRsults);
}
}
}
@@ -1123,9 +1123,9 @@ Some processes occur double to document the complex index migration structure.==
This is the list of web pages that this peer initiated to crawl,==Dies ist eine Liste von Internetseiten, bei der Ihr Peer den Crawl initiiert hat,
but had been crawled by <em>other</em> peers.==die aber von <em>anderen</em> Peers gecrawlt wurden.
This is the 'mirror'-case of process (6).==Dies ist der 'Gegensatz' Prozess zu (6)
<em>Use Case:</em> You get entries here, if you start a local crawl on the 'Index Creation'-Page and check the==<em>Anwendungsfall:</em> Sie erhalten hier Einträge, wenn Sie einen lokalen Crawl auf der 'Index erzeugen' Seite starten und
'Do Remote Indexing'-flag. Every page that a remote peer indexes upon this peer's request==die 'Remote Indexierung' aktivieren. Jede Seite, die von einem remote Peer durch Ihre Anfrage indexiert wurde,
is reported back and can be monitored here.==wird nun zurück gemeldet und hier angezeigt.
<em>Use Case:</em> You get entries here, if you start a local crawl on the '<a href="CrawlStartExpert.html">Advanced Crawler</a>' page and check the==<em>Anwendungsfall:</em> Sie erhalten hier Einträge, wenn Sie einen lokalen Crawl auf der '<a href="CrawlStartExpert.html">Experten Crawl Start</a>' Seite starten und
'Do Remote Indexing'-flag, and if you checked the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page.==die 'Remote Indexierung' aktivieren, und wenn Sie auf der Seite '<a href="RemoteCrawl_p.html">Remote Crawling</a>' das Kennzeichen'Akzeptiere Remote Crawl Anfragen' gesetzt haben.
Every page that a remote peer indexes upon this peer's request is reported back and can be monitored here.==Jede Seite, die von einem remote Peer durch Ihre Anfrage indexiert wurde, wird nun zurück gemeldet und hier angezeigt.
(2) Results for Result of Search Queries==(2) Ergebnisse der Resultate bei Suchanfragen
This index transfer was initiated by your peer by doing a search query.==Dieser Index Transfer wurde von Ihrem Peer dadurch initiiert, dass Sie eine Suchanfrage gestartet haben.
The index was crawled and contributed by other peers.==Der Index wurde von anderen Peers gecrawlt und nun Ihnen breitgestellt.
@@ -1149,7 +1149,7 @@ These web pages had been crawled by your own crawl task.==Diese Internetseiten w
(6) Results for Global Crawling==(6) Ergebnisse des globalen Crawlens
These pages had been indexed by your peer, but the crawl was initiated by a remote peer.==Diese Seiten wurden von Ihrem Peer indexiert, der Crawl wurde aber von einem anderen Peer initiiert (remote-Crawl).
This is the 'mirror'-case of process (1).==Dies ist der 'Gegensatz' Prozess zu (1).
<em>Use Case:</em> This list may fill if you check the 'Accept remote crawling requests'-flag on the 'Index Crate' page==<em>Anwendung:</em> Diese Liste füllt sich, wenn Sie 'Akzeptiere remote Crawling Anfragen' auf der 'Index erzeugen' Seite aktiviert haben.
<em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page==<em>Anwendung:</em> Diese Liste füllt sich, wenn Sie 'Akzeptiere Remote Crawl Anfragen' auf der '<a href="RemoteCrawl_p.html">Remote Crawling</a>' Seite aktiviert haben.
The stack is empty.==Die Liste ist leer.
Statistics about #[domains]# domains in this stack:==Statistiken über #[domains]# Domains in diesem Bereich:
(7) Results from surrogates import==(7) Ergebnisse aus dem Surrogat Import
@@ -892,9 +892,9 @@ Some processes occur double to document the complex index migration structure.==
This is the list of web pages that this peer initiated to crawl,==यह इस सहकर्मी क्रॉल करने के लिए शुरू की है कि वेब पृष्ठों की सूची है,
but had been crawled by <em>other</em> peers.==लेकिन <em> अन्य </ em> साथियों द्वारा क्रॉल किया गया था.
This is the 'mirror'-case of process (6).==इस प्रक्रिया के 'mirror' मामले है (6 )
<em>Use Case:</em> You get entries here, if you start a local crawl on the 'Index Creation'-Page and check the==<em> प्रयोग करें प्रकरण: </ em> आप 'सूचकांक Creation'-पृष्ठ पर एक स्थानीय क्रॉल शुरू, यहाँ प्रविष्टियों मिलता है और जाँच
'Do Remote Indexing'-flag. Every page that a remote peer indexes upon this peer's request=='Indexing' झंडा रिमोट करो. हर पृष्ठ कि इस सहकर्मी के अनुरोध पर एक दूरस्थ सहकर्मी अनुक्रमित
is reported back and can be monitored here.==वापस सूचना दी है और यहाँ पर नजर रखी जा सकती है.
<em>Use Case:</em> You get entries here, if you start a local crawl on the '<a href="CrawlStartExpert.html">Advanced Crawler</a>' page and check the==<em> प्रयोग करें प्रकरण: </ em> आप '<a href="CrawlStartExpert.html">सूचकांक Creation</a>' page पृष्ठ पर एक स्थानीय क्रॉल शुरू, यहाँ प्रविष्टियों मिलता है और जाँच
'Do Remote Indexing'-flag, and if you checked the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page.=='Indexing' झंडा रिमोट करो, and if you checked the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page.
Every page that a remote peer indexes upon this peer's request is reported back and can be monitored here.==हर पृष्ठ कि इस सहकर्मी के अनुरोध पर एक दूरस्थ सहकर्मी अनुक्रमित वापस सूचना दी है और यहाँ पर नजर रखी जा सकती है.
(2) Results for Result of Search Queries==(2) खोज प्रश्नों का रिजल्ट के लिए परिणाम
This index transfer was initiated by your peer by doing a search query.==इस सूचकांक में स्थानांतरण एक खोज क्वेरी कर रही द्वारा अपने साथियों के द्वारा शुरू किया गया था
The index was crawled and contributed by other peers.==सूचकांक अन्य साथियों द्वारा क्रॉल और योगदान था.
@@ -918,7 +918,7 @@ These web pages had been crawled by your own crawl task.==इन वेब प
(6) Results for Global Crawling==(6) ग्लोबल रेंगने के लिए परिणाम
These pages had been indexed by your peer, but the crawl was initiated by a remote peer.==इन पृष्ठों के अपने साथियों द्वारा अनुक्रमित किया गया था, लेकिन क्रॉल एक दूरस्थ सहकर्मी से शुरू किया गया था
This is the 'mirror'-case of process (1).==इस प्रक्रिया के 'mirror' मामले है (1).
<em>Use Case:</em> This list may fill if you check the 'Accept remote crawling requests'-flag on the 'Index Crate' page==<em> प्रयोग करें प्रकरण: </ em> आप जाँच अगर इस सूची में भर सकते हैं 'स्वीकार रिमोट पर requests' झंडा रेंगने' सूचकांक टोकरा 'पेज
<em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the 'Index Crate' page==<em> प्रयोग करें प्रकरण: </ em> आप जाँच अगर इस सूची में भर सकते हैं 'स्वीकार रिमोट पर requests' झंडा रेंगने' सूचकांक टोकरा 'पेज
The stack is empty.==ढेर खाली है.
Statistics about #[domains]# domains in this stack:==इस ढेर में # [डोमेन ] # डोमेन के बारे में सांख्यिकी:
(7) Results from surrogates import==(7) Surrogates के आयात से का परिणाम
@@ -2932,13 +2932,13 @@
<source>This is the 'mirror'-case of process (6).</source>
</trans-unit>
<trans-unit id="477bfd45" xml:space="preserve" approved="no" translate="yes">
<source>&lt;em&gt;Use Case:&lt;/em&gt; You get entries here, if you start a local crawl on the 'Index Creation'-Page and check the</source>
<source>&lt;em&gt;Use Case:&lt;/em&gt; You get entries here, if you start a local crawl on the '&lt;a href="CrawlStartExpert.html"&gt;Advanced Crawler&lt;/a&gt;' page and check the</source>
</trans-unit>
<trans-unit id="59986be9" xml:space="preserve" approved="no" translate="yes">
<source>'Do Remote Indexing'-flag. Every page that a remote peer indexes upon this peer's request</source>
<source>'Do Remote Indexing'-flag, and if you checked the 'Accept Remote Crawl Requests'-flag on the '&lt;a href="RemoteCrawl_p.html"&gt;Remote Crawling&lt;/a&gt;' page.</source>
</trans-unit>
<trans-unit id="7f968d75" xml:space="preserve" approved="no" translate="yes">
<source>is reported back and can be monitored here.</source>
<source>Every page that a remote peer indexes upon this peer's request is reported back and can be monitored here.</source>
</trans-unit>
<trans-unit id="c4ada7d6" xml:space="preserve" approved="no" translate="yes">
<source>(2) Results for Result of Search Queries</source>
@@ -3007,7 +3007,7 @@
<source>This is the 'mirror'-case of process (1).</source>
</trans-unit>
<trans-unit id="7d5fcb88" xml:space="preserve" approved="no" translate="yes">
<source>&lt;em&gt;Use Case:&lt;/em&gt; This list may fill if you check the 'Accept remote crawling requests'-flag on the '&lt;a href="RemoteCrawl_p.html"&gt;Index Create&lt;/a&gt;' page</source>
<source>&lt;em&gt;Use Case:&lt;/em&gt; This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '&lt;a href="RemoteCrawl_p.html"&gt;Remote Crawling&lt;/a&gt;' page</source>
</trans-unit>
<trans-unit id="fa27dad0" xml:space="preserve" approved="no" translate="yes">
<source>The stack is empty.</source>

0 comments on commit 534f09e

Please sign in to comment.
You can’t perform that action at this time.