Skip to content

Commit

Permalink
fix for filetype naviagtor
Browse files Browse the repository at this point in the history
  • Loading branch information
@Orbiter
Orbiter committed Nov 7, 2012
1 parent 71ed8e5 commit 570e42c
Show file tree
Hide file tree
Showing 7 changed files with 29 additions and 64 deletions.
23 changes: 13 additions & 10 deletions htroot/suggest.java
View file
Expand Up @@ -22,6 +22,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA


import java.util.ConcurrentModificationException;
import java.util.Iterator;

import net.yacy.cora.protocol.HeaderFramework;
Expand Down Expand Up @@ -72,16 +73,18 @@ public static serverObjects respond(final RequestHeader header, final serverObje
String suggestion;
//[#[query]#,[#{suggestions}##[text]##(eol)#,::#(/eol)##{/suggestions}#]]
while (c < meanMax && meanIt.hasNext()) {
suggestion = meanIt.next().toString();
if (json) {
prop.putJSON("suggestions_" + c + "_text", suggestion);
} else if (xml) {
prop.putXML("suggestions_" + c + "_text", suggestion);
} else {
prop.putHTML("suggestions_" + c + "_text", suggestion);
}
prop.put("suggestions_" + c + "_eol", 0);
c++;
try {
suggestion = meanIt.next().toString();
if (json) {
prop.putJSON("suggestions_" + c + "_text", suggestion);
} else if (xml) {
prop.putXML("suggestions_" + c + "_text", suggestion);
} else {
prop.putHTML("suggestions_" + c + "_text", suggestion);
}
prop.put("suggestions_" + c + "_eol", 0);
c++;
} catch (ConcurrentModificationException e) {}
}
}

Expand Down
11 changes: 5 additions & 6 deletions htroot/yacysearch.java
View file
Expand Up @@ -374,15 +374,14 @@ public static serverObjects respond(
ranking.coeff_date = RankingProfile.COEFF_MAX;
modifier.append("/date ");
}
if ( querystring.indexOf("/http", 0) >= 0 ) {
querystring = querystring.replace("/http", "");
urlmask = "https?://.*";
modifier.append("/http ");
}
if ( querystring.indexOf("/https", 0) >= 0 ) {
querystring = querystring.replace("/https", "");
urlmask = "https?://.*";
urlmask = "https://.*";
modifier.append("/https ");
} else if ( querystring.indexOf("/http", 0) >= 0 ) {
querystring = querystring.replace("/http", "");
urlmask = "http://.*";
modifier.append("/http ");
}
if ( querystring.indexOf("/ftp", 0) >= 0 ) {
querystring = querystring.replace("/ftp", "");
Expand Down
3 changes: 2 additions & 1 deletion source/net/yacy/cora/sorting/AbstractScoreMap.java
View file
Expand Up @@ -36,7 +36,8 @@ public abstract class AbstractScoreMap<E> implements ScoreMap<E> {
public void inc(ScoreMap<E> map) {
if (map == null) return;
for (E entry: map) {
this.inc(entry, map.get(entry));
int count = map.get(entry);
if (count > 0) this.inc(entry, count);
}
}

Expand Down
12 changes: 0 additions & 12 deletions source/net/yacy/kelondro/data/meta/DigestURI.java
View file
Expand Up @@ -270,18 +270,6 @@ private final byte[] urlHashComputation() {
return b;
}

/**
* return true if the protocol of the URL was 'http'
* this is not true if the protocol was 'https'
* @param hash
* @return true for url hashes that point to http services; false otherwise
*/
public static final boolean flag4HTTPset(final byte[] hash) {
assert hash.length == 12;
final byte flagbyte = hash[11];
return (flagbyte & 32) == 1;
}

private static char subdomPortPath(final String subdom, final int port, final String rootpath) {
final StringBuilder sb = new StringBuilder(subdom.length() + rootpath.length() + 8);
sb.append(subdom).append(':').append(Integer.toString(port)).append(':').append(rootpath);
Expand Down
14 changes: 9 additions & 5 deletions source/net/yacy/search/query/QueryParams.java
View file
Expand Up @@ -533,17 +533,21 @@ public SolrQuery solrQuery() {
q.append(" AND ").append(YaCySchema.host_id_s.getSolrFieldName()).append(":\"").append(this.nav_sitehash).append('\"');
}
String urlMaskPattern = this.urlMask.pattern();

// translate filetype navigation
int extm = urlMaskPattern.indexOf(".*\\.");
if (extm >= 0) {
String ext = urlMaskPattern.substring(extm + 4);
q.append(" AND ").append(YaCySchema.url_file_ext_s.getSolrFieldName()).append(':').append(ext);
}
extm = urlMaskPattern.indexOf("?://.*");
if (extm >= 0) {
String protocol = urlMaskPattern.substring(0, extm);
q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append(protocol);
}

// translate protocol navigation
if (urlMaskPattern.startsWith("http://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("http");
else if (urlMaskPattern.startsWith("https://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("https");
else if (urlMaskPattern.startsWith("ftp://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("ftp");
else if (urlMaskPattern.startsWith("smb://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("smb");
else if (urlMaskPattern.startsWith("file://.*")) q.append(" AND ").append(YaCySchema.url_protocol_s.getSolrFieldName()).append(':').append("file");

// construct query
final SolrQuery params = new SolrQuery();
params.setQuery(q.toString());
Expand Down
11 changes: 0 additions & 11 deletions source/net/yacy/search/query/RankingProcess.java
View file
Expand Up @@ -51,7 +51,6 @@
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.Condenser;
import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
Expand Down Expand Up @@ -290,9 +289,6 @@ public void add(
long timeout = System.currentTimeMillis() + maxtime;
try {
WordReferenceVars iEntry;
final String pattern = this.query.urlMask.pattern();
final boolean httpPattern = pattern.equals("http://.*");
final boolean noHttpButProtocolPattern = pattern.equals("https://.*") || pattern.equals("ftp://.*") || pattern.equals("smb://.*") || pattern.equals("file://.*");
long remaining;
pollloop: while ( true ) {
remaining = timeout - System.currentTimeMillis();
Expand Down Expand Up @@ -347,13 +343,6 @@ public void add(
this.hostHashNavigator.inc(hosthash);
this.hostHashResolver.put(hosthash, iEntry.urlhash());

// check protocol
if (!this.query.urlMask_isCatchall) {
final boolean httpFlagSet = DigestURI.flag4HTTPset(iEntry.urlHash);
if (httpPattern && !httpFlagSet) continue pollloop;
if (noHttpButProtocolPattern && httpFlagSet) continue pollloop;
}

// check vocabulary constraint
String subject = YaCyMetadata.hashURI(iEntry.urlhash());
Resource resource = JenaTripleStore.getResource(subject);
Expand Down
19 changes: 0 additions & 19 deletions source/net/yacy/search/query/SearchEvent.java
View file
Expand Up @@ -62,7 +62,6 @@
import net.yacy.document.Condenser;
import net.yacy.document.LargeNumberCache;
import net.yacy.interaction.contentcontrol.ContentControlFilterUpdateThread;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReference;
Expand Down Expand Up @@ -469,9 +468,6 @@ public void add(

// apply all constraints
try {
final String pattern = this.query.urlMask.pattern();
final boolean httpPattern = pattern.equals("http://.*");
final boolean noHttpButProtocolPattern = pattern.equals("https://.*") || pattern.equals("ftp://.*") || pattern.equals("smb://.*") || pattern.equals("file://.*");
pollloop: for (URIMetadataNode iEntry: index) {

if ( !this.query.urlMask_isCatchall ) {
Expand Down Expand Up @@ -515,17 +511,6 @@ public void add(
if (!hosthash.equals(this.query.nav_sitehash)) continue pollloop;
}

// check protocol
if ( !this.query.urlMask_isCatchall ) {
final boolean httpFlagSet = DigestURI.flag4HTTPset(iEntry.hash());
if ( httpPattern && !httpFlagSet ) {
continue pollloop;
}
if ( noHttpButProtocolPattern && httpFlagSet ) {
continue pollloop;
}
}

// check vocabulary constraint
String subject = YaCyMetadata.hashURI(iEntry.hash());
Resource resource = JenaTripleStore.getResource(subject);
Expand Down Expand Up @@ -850,10 +835,6 @@ public URIMetadataNode takeURL(final boolean skipDoubleDom, final long waitingti
}
}

// protocol navigation
final String protocol = page.url().getProtocol();
this.protocolNavigator.inc(protocol);

return page; // accept url
}
Log.logWarning("RWIProcess", "loop terminated");
Expand Down

0 comments on commit 570e42c

Please sign in to comment.