Skip to content

Commit

Permalink
Merge pull request #1 from yacy/master
Browse files Browse the repository at this point in the history 
update
  • Loading branch information
@otteresk
otteresk committed Sep 27, 2017
2 parents 66cb9c4 + fc28c58 commit 9f0e7dc
Show file tree
Hide file tree
Showing 50 changed files with 1,116 additions and 299 deletions.
37 changes: 18 additions & 19 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,24 +65,23 @@
<classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
<classpathentry kind="lib" path="lib/commons-math3-3.4.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/solr-core-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/solr-solrj-6.6.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-extras-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/solr-core-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/solr-solrj-6.6.1.jar"/>
<classpathentry kind="lib" path="lib/zookeeper-3.4.10.jar"/>
<classpathentry kind="lib" path="lib/metrics-core-3.2.2.jar"/>
<classpathentry kind="lib" path="lib/spatial4j-0.6.jar"/>
Expand All @@ -99,7 +98,7 @@
<classpathentry kind="lib" path="lib/imageio-metadata-3.3.1.jar"/>
<classpathentry kind="lib" path="lib/imageio-tiff-3.3.1.jar"/>
<classpathentry kind="lib" path="lib/imageio-bmp-3.3.1.jar"/>
<classpathentry kind="lib" path="lib/jsonic-1.2.0.jar"/>
<classpathentry kind="lib" path="lib/jsonic-1.3.10.jar"/>
<classpathentry kind="lib" path="lib/langdetect.jar"/>
<classpathentry kind="lib" path="lib/jwat-common-1.1.0.jar"/>
<classpathentry kind="lib" path="lib/jwat-gzip-1.1.0.jar"/>
Expand Down
37 changes: 18 additions & 19 deletions build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,29 +218,28 @@
<pathelement location="${lib}/jetty-xml-9.4.6.v20170531.jar" />
<pathelement location="${lib}/jsch-0.1.54.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsonic-1.2.0.jar" />
<pathelement location="${lib}/jsonic-1.3.10.jar" />
<pathelement location="${lib}/jsoup-1.10.3.jar" />
<pathelement location="${lib}/jwat-archive-common-1.1.0.jar" />
<pathelement location="${lib}/jwat-common-1.1.0.jar" />
<pathelement location="${lib}/jwat-gzip-1.1.0.jar" />
<pathelement location="${lib}/jwat-warc-1.1.0.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.24.jar" />
<pathelement location="${lib}/lucene-analyzers-common-6.6.0.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-6.6.0.jar" />
<pathelement location="${lib}/lucene-backward-codecs-6.6.0.jar" />
<pathelement location="${lib}/lucene-classification-6.6.0.jar" />
<pathelement location="${lib}/lucene-codecs-6.6.0.jar" />
<pathelement location="${lib}/lucene-core-6.6.0.jar" />
<pathelement location="${lib}/lucene-facet-6.6.0.jar" />
<pathelement location="${lib}/lucene-grouping-6.6.0.jar" />
<pathelement location="${lib}/lucene-highlighter-6.6.0.jar" />
<pathelement location="${lib}/lucene-join-6.6.0.jar" />
<pathelement location="${lib}/lucene-memory-6.6.0.jar" />
<pathelement location="${lib}/lucene-misc-6.6.0.jar" />
<pathelement location="${lib}/lucene-queries-6.6.0.jar" />
<pathelement location="${lib}/lucene-queryparser-6.6.0.jar" />
<pathelement location="${lib}/lucene-spatial-6.6.0.jar" />
<pathelement location="${lib}/lucene-suggest-6.6.0.jar" />
<pathelement location="${lib}/lucene-analyzers-common-6.6.1.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-6.6.1.jar" />
<pathelement location="${lib}/lucene-backward-codecs-6.6.1.jar" />
<pathelement location="${lib}/lucene-classification-6.6.1.jar" />
<pathelement location="${lib}/lucene-codecs-6.6.1.jar" />
<pathelement location="${lib}/lucene-core-6.6.1.jar" />
<pathelement location="${lib}/lucene-grouping-6.6.1.jar" />
<pathelement location="${lib}/lucene-highlighter-6.6.1.jar" />
<pathelement location="${lib}/lucene-join-6.6.1.jar" />
<pathelement location="${lib}/lucene-memory-6.6.1.jar" />
<pathelement location="${lib}/lucene-misc-6.6.1.jar" />
<pathelement location="${lib}/lucene-queries-6.6.1.jar" />
<pathelement location="${lib}/lucene-queryparser-6.6.1.jar" />
<pathelement location="${lib}/lucene-spatial-extras-6.6.1.jar" />
<pathelement location="${lib}/lucene-suggest-6.6.1.jar" />
<pathelement location="${lib}/metadata-extractor-2.10.1.jar" />
<pathelement location="${lib}/metrics-core-3.2.2.jar" />
<pathelement location="${lib}/noggit-0.6.jar" />
Expand All @@ -251,8 +250,8 @@
<pathelement location="${lib}/poi-scratchpad-3.16.jar" />
<pathelement location="${lib}/slf4j-api-1.7.24.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.24.jar" />
<pathelement location="${lib}/solr-core-6.6.0.jar" />
<pathelement location="${lib}/solr-solrj-6.6.0.jar" />
<pathelement location="${lib}/solr-core-6.6.1.jar" />
<pathelement location="${lib}/solr-solrj-6.6.1.jar" />
<pathelement location="${lib}/spatial4j-0.6.jar" />
<pathelement location="${lib}/stax2-api_3.1.4.jar" />
<pathelement location="${lib}/weupnp-0.1.4.jar" />
Expand Down
8 changes: 8 additions & 0 deletions defaults/yacy.init
View file
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,9 @@ search.navigation=location,hosts,authors,namespace,topics,filetype,protocol,lang
# max number of items displayed in search navigators
search.navigation.maxcount=100

# max number of items displayed in the dates navigator
search.navigation.dates.maxcount=640

# search result verification and snippet fetch caching rules
# each search result can be verified byloading the link from the web
# this can be enhanced using a cache. In some cases it may be appropriate
Expand All @@ -907,6 +910,11 @@ search.excludehosth=
# the cases of nocache, iffresh and ifexist causes an index deletion
search.verify.delete = true

# If enabled, the results are sorted in the browser using Javascript.
# This usually improves ranking accuracy, but doesn't work well for users
# who have Javascript disabled, are using screen readers, or are on slow computers.
search.jsresort = false

# remote search details
remotesearch.maxcount = 10
remotesearch.maxtime = 3000
Expand Down
10 changes: 10 additions & 0 deletions htroot/ConfigPortal_p.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ <h2>Integration of a Search Portal</h2>
<input type="radio" name="search.options" value="false" #(search.options)#checked="checked"::#(/search.options)# />do not show Advanced Search
</dd>

<dt>Remote results resorting</dt>
<dd>
<label title="Remote results resorting can be triggered once the 'Refresh sorting' button (near the 'Search' button) becomes available.">
<input type="radio" name="search.jsresort" value="false" #(search.jsresort)#checked="checked"::#(/search.jsresort)# />On demand, server-side
</label>
<label title="This usually improves ranking accuracy, but doesn't work well for users who have Javascript disabled, are using screen readers, or are on slow computers.">
<input type="radio" name="search.jsresort" value="true" #(search.jsresort)#::checked="checked"#(/search.jsresort)# />Automated, with JavaScript in the browser <strong>for authenticated users only</strong>.
</label>
</dd>

<dt>Snippet Fetch Strategy &amp; Link Verification</dt>
<dd>
<img src="env/grafics/idea.png" width="32" height="32" alt="idea" align="center"/>Speed up search results with this option! (use CACHEONLY or FALSE to switch off verification)<br/>
Expand Down
22 changes: 22 additions & 0 deletions htroot/ConfigPortal_p.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.http.HTTPDFileHandler;
Expand All @@ -54,6 +55,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);

boolean cleanSearchCache = false;

if (post.containsKey("popup")) {
final String popup = post.get("popup", "status");
if ("front".equals(popup)) {
Expand Down Expand Up @@ -87,6 +90,12 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu"));
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE));
sb.setConfig("search.options", post.getBoolean("search.options"));

final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
final boolean newJsResort = post.getBoolean(SwitchboardConstants.SEARCH_JS_RESORT);
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = oldJsResort != newJsResort;
sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);

sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE));

Expand Down Expand Up @@ -147,6 +156,14 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true"));
sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics"));
sb.setConfig("search.options", config.getProperty("search.options","true"));

final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
final boolean newJsResort = Boolean.parseBoolean(config.getProperty(SwitchboardConstants.SEARCH_JS_RESORT, String.valueOf(SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT)));
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = oldJsResort != newJsResort;
sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);


sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE));
Expand All @@ -157,6 +174,10 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts",""));
sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth",""));
}

if(cleanSearchCache) {
SearchEventCache.cleanupEvents(true);
}
}

/* Acquire a transaction token for the next POST form submission */
Expand All @@ -171,6 +192,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
prop.put(SwitchboardConstants.PUBLIC_SEARCHPAGE, sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, false) ? 1 : 0);
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
prop.put(SwitchboardConstants.SEARCH_JS_RESORT, sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT) ? 1 : 0);

prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0);
prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0"));
Expand Down
Loading

0 comments on commit 9f0e7dc

Please sign in to comment.