From 9b4c699526c404ae18173faf618c27a7e8de5cb3 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 31 May 2012 22:39:53 +0200 Subject: [PATCH] ehanced location search: - search request are now made using a map boundary - search results are only computed for the map boundary - the number of results is adopted to the results in the visible range - added a double-buffering for the search result markers - added a search query option for the search results: /radius/// --- htroot/yacy/search.java | 4 +- htroot/yacysearch.java | 26 ++++++- htroot/yacysearch_location.html | 20 ++++-- htroot/yacysearch_location.java | 5 +- source/de/anomic/crawler/ResultURLs.java | 2 +- source/de/anomic/server/serverObjects.java | 14 ++++ .../cora/services/federated/solr/SolrDoc.java | 4 ++ source/net/yacy/document/Document.java | 12 ++-- source/net/yacy/document/content/DCEntry.java | 70 +++++++++---------- .../yacy/document/content/dao/PhpBB3Dao.java | 2 +- .../document/parser/html/ContentScraper.java | 16 ++--- .../yacy/kelondro/data/meta/URIMetadata.java | 24 +++---- .../kelondro/data/meta/URIMetadataRow.java | 24 +++---- source/net/yacy/search/index/Segment.java | 2 +- .../yacy/search/index/SolrConfiguration.java | 4 ++ source/net/yacy/search/query/QueryParams.java | 16 ++++- source/net/yacy/search/query/RWIProcess.java | 12 ++++ .../net/yacy/search/query/SnippetProcess.java | 4 +- .../net/yacy/search/snippet/ResultEntry.java | 4 +- 19 files changed, 173 insertions(+), 92 deletions(-) diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 38258a5b98..d056b81c9c 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -255,7 +255,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje indexSegment, rankingProfile, header.get(RequestHeader.USER_AGENT, ""), - false + false, 0.0d, 0.0d, 0.0d ); Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links"); @@ -318,7 +318,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje sb.indexSegments.segment(Segments.Process.PUBLIC), rankingProfile, header.get(RequestHeader.USER_AGENT, ""), - false + false, 0.0d, 0.0d, 0.0d ); Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links"); EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), "")); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index bb345cd81b..6a4e6da269 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -485,6 +485,29 @@ public static serverObjects respond( } } + int radius = 0; + double lon = 0.0d, lat = 0.0d, rad = 0.0d; + if ((radius = querystring.indexOf("/radius/")) >= 0) { + int ve = querystring.indexOf(' ', radius + 8); + String geo = ""; + if (ve < 0) { + geo = querystring.substring(radius); + querystring = querystring.substring(0, radius).trim(); + } else { + geo = querystring.substring(radius, ve); + querystring = querystring.substring(0, radius) + querystring.substring(ve); + } + geo = geo.substring(8); + String[] sp = geo.split("/"); + if (sp.length == 3) try { + lat = Double.parseDouble(sp[0]); + lon = Double.parseDouble(sp[1]); + rad = Double.parseDouble(sp[2]); + } catch (NumberFormatException e) { + lon = 0.0d; lat = 0.0d; rad = 0.0d; + } + } + String tenant = null; if ( post.containsKey("tenant") ) { tenant = post.get("tenant"); @@ -757,7 +780,8 @@ public static serverObjects respond( header.get(RequestHeader.USER_AGENT, ""), sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) - && sb.peers.mySeed().getFlagAcceptRemoteIndex()); + && sb.peers.mySeed().getFlagAcceptRemoteIndex(), + lat, lon, rad); EventTracker.delete(EventTracker.EClass.SEARCH); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch( theQuery.id(true), diff --git a/htroot/yacysearch_location.html b/htroot/yacysearch_location.html index ee2c33921e..4e7189a657 100644 --- a/htroot/yacysearch_location.html +++ b/htroot/yacysearch_location.html @@ -20,6 +20,8 @@ var shallZoom = true; // flag if the map shall be zoomed to the search result var shallReload = false; // flag if the search shall be repeated upon next resize of the window var poisfound = 0; + var kmNormal = 100; // ca. 1km grid for radius + var meterNormal = 100000; // ca. 1m grid for location // automatically adapt height of map upon window resize function adaptHeight() { @@ -66,6 +68,7 @@ function zoomChanged () { shallZoom = false; // no automatic zoom if user hs zoomed + refreshsearch(); } // called if a search is submitted @@ -81,8 +84,11 @@ var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326")); var extend = map.getExtent().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326")); var radius = Math.sqrt(extend.getWidth() * extend.getWidth() + extend.getHeight() * extend.getHeight()) / 2; - searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co}); - searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md}); + radius = Math.floor(radius * kmNormal + 1) / kmNormal; + var lon = Math.floor(center.lon * meterNormal) / meterNormal; + var lat = Math.floor(center.lat * meterNormal) / meterNormal; + searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co}); + searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md}); map.addLayer(searchLayer_co0); map.addLayer(searchLayer_md0); //document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query); @@ -92,7 +98,7 @@ function refresh() { // check double-buffer state - if (searchLayer_md0 != null && searchLayer_md0.markers.length > 0) { + if (searchLayer_md0 != null) { // switch the double-buffer if (searchLayer_md1 != null) searchLayer_md1.destroy(); if (searchLayer_co1 != null) searchLayer_co1.destroy(); @@ -108,13 +114,13 @@ } var cocount = searchLayer_co1.markers.length; var mdcount = searchLayer_md1.markers.length; - if (mdcount + cocount <= poisfound) { - return; - } + //if (mdcount + cocount <= poisfound) { + // return; + //} // update map and result line poisfound = mdcount + cocount; - document.getElementById('resultline').innerHTML = poisfound + " POIs found"; + document.getElementById('resultline').innerHTML = poisfound + " POIs found in visible map"; // check if any of the markers is visible on the screen. // this is only necessary if shallZoom = false since this would set shallZoom = true diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 5a147fd7a8..0283622205 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -60,6 +60,9 @@ public static serverObjects respond(final RequestHeader header, final serverObje final boolean search_subject = alltext || post.get("dom", "").indexOf("subject",0) >= 0; final long maximumTime = post.getLong("maximumTime", 5000); final int maximumRecords = post.getInt("maximumRecords", 3000); + final double lon = post.getDouble("lon", 0.0d); + final double lat = post.getDouble("lat", 0.0d); + final double radius = post.getDouble("r", 0.0d); //i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100 int placemarkCounter = 0; @@ -89,7 +92,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje // get a queue of search results final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss"; final BlockingQueue results = new LinkedBlockingQueue(); - SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null); + SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, null); // take the results and compute some locations RSSMessage message; diff --git a/source/de/anomic/crawler/ResultURLs.java b/source/de/anomic/crawler/ResultURLs.java index 0eeb89c793..14d818e305 100644 --- a/source/de/anomic/crawler/ResultURLs.java +++ b/source/de/anomic/crawler/ResultURLs.java @@ -222,7 +222,7 @@ public static boolean remove(final String urlHash) { public static void main(final String[] args) { try { final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/"); - final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0); + final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0); final EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING; System.out.println("valid test:\n======="); // add diff --git a/source/de/anomic/server/serverObjects.java b/source/de/anomic/server/serverObjects.java index b7b25b9897..836a96d43c 100644 --- a/source/de/anomic/server/serverObjects.java +++ b/source/de/anomic/server/serverObjects.java @@ -183,6 +183,10 @@ public double put(final String key, final float value) { return (null == this.put(key, Float.toString(value))) ? Float.NaN : value; } + public double put(final String key, final double value) { + return (null == this.put(key, Double.toString(value))) ? Double.NaN : value; + } + /** * same as {@link #put(String, double)} but for integer types * @return Returns 0 for the error case. @@ -359,6 +363,16 @@ public float getFloat(final String key, final float dflt) { } } + public double getDouble(final String key, final double dflt) { + final String s = removeByteOrderMark(super.get(key)); + if (s == null) return dflt; + try { + return Double.parseDouble(s); + } catch (final NumberFormatException e) { + return dflt; + } + } + public boolean getBoolean(final String key, final boolean dflt) { String s = removeByteOrderMark(super.get(key)); if (s == null) return dflt; diff --git a/source/net/yacy/cora/services/federated/solr/SolrDoc.java b/source/net/yacy/cora/services/federated/solr/SolrDoc.java index fba332073b..ee43f61cae 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrDoc.java +++ b/source/net/yacy/cora/services/federated/solr/SolrDoc.java @@ -64,6 +64,10 @@ public final void addSolr(final SolrField key, final float value) { this.setField(key.getSolrFieldName(), value); } + public final void addSolr(final SolrField key, final double value) { + this.setField(key.getSolrFieldName(), value); + } + public final void addSolr(final SolrField key, final boolean value) { this.setField(key.getSolrFieldName(), value); } diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index c323a8be96..55a92ef392 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -87,7 +87,7 @@ public class Document { private boolean resorted; private final Set languages; private final boolean indexingDenied; - private final float lon, lat; + private final double lon, lat; private final Object parserObject; // the source object that was used to create the Document public Document(final MultiProtocolURI location, final String mimeType, final String charset, @@ -95,7 +95,7 @@ public Document(final MultiProtocolURI location, final String mimeType, final St final Set languages, final String[] keywords, final String title, final String author, final String publisher, final String[] sections, final String abstrct, - final float lon, final float lat, + final double lon, final double lat, final Object text, final Map anchors, final Map rss, @@ -400,11 +400,11 @@ public Map getEmaillinks() { return this.emaillinks; } - public float lon() { + public double lon() { return this.lon; } - public float lat() { + public double lat() { return this.lat; } @@ -743,7 +743,7 @@ public static Document mergeDocuments(final MultiProtocolURI location, final Map anchors = new HashMap(); final Map rss = new HashMap(); final Map images = new HashMap(); - float lon = 0.0f, lat = 0.0f; + double lon = 0.0d, lat = 0.0d; for (final Document doc: docs) { @@ -784,7 +784,7 @@ public static Document mergeDocuments(final MultiProtocolURI location, anchors.putAll(doc.getAnchors()); rss.putAll(doc.getRSS()); ContentScraper.addAllImages(images, doc.getImages()); - if (doc.lon() != 0.0f && doc.lat() != 0.0f) { lon = doc.lon(); lat = doc.lat(); } + if (doc.lon() != 0.0d && doc.lat() != 0.0d) { lon = doc.lon(); lat = doc.lat(); } } return new Document( location, diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java index c5d135c56f..124e311a1e 100644 --- a/source/net/yacy/document/content/DCEntry.java +++ b/source/net/yacy/document/content/DCEntry.java @@ -7,7 +7,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -43,9 +43,9 @@ public class DCEntry extends TreeMap { - + private static final long serialVersionUID = -2050291583515701559L; - + // use a collator to relax when distinguishing between lowercase und uppercase letters private static final Collator insensitiveCollator = Collator.getInstance(Locale.US); static { @@ -53,19 +53,19 @@ public class DCEntry extends TreeMap { insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION); } public static final DCEntry poison = new DCEntry(); - + public DCEntry() { super((Collator) insensitiveCollator.clone()); } - + public DCEntry( DigestURI url, Date date, String title, String author, String body, - float lat, - float lon + double lat, + double lon ) { super((Collator) insensitiveCollator.clone()); this.put("dc:identifier", url.toNormalform(true, false)); @@ -73,10 +73,10 @@ public DCEntry( this.put("dc:title", title); this.put("dc:creator", author); this.put("dc:description", body); - this.put("geo:lat", Float.toString(lat)); - this.put("geo:long", Float.toString(lon)); + this.put("geo:lat", Double.toString(lat)); + this.put("geo:long", Double.toString(lon)); } - + /* DC according to rfc 5013 @@ -108,7 +108,7 @@ public Date getDate() { return new Date(); } } - + public DigestURI getIdentifier(boolean useRelationAsAlternative) { String u = this.get("url"); if (u == null) u = this.get("dc:identifier"); @@ -130,7 +130,7 @@ public DigestURI getIdentifier(boolean useRelationAsAlternative) { return null; } } - + public DigestURI getRelation() { String u = this.get("dc:relation"); if (u == null) return null; @@ -146,7 +146,7 @@ public DigestURI getRelation() { return null; } } - + private String bestU(String[] urls) { for (String uu: urls) { if (uu.startsWith("http://") && (uu.endsWith(".html") || uu.endsWith(".htm") || uu.endsWith(".pdf") || uu.endsWith(".doc") || uu.endsWith(".rss") || uu.endsWith(".xml"))) return uu; @@ -164,7 +164,7 @@ private String bestU(String[] urls) { } return urls[0]; } - + //modified by copperdust; Ukraine, 2012 public String getLanguage() {//final language computation String l = this.get("dc:language");//from document metainfo @@ -172,31 +172,31 @@ public String getLanguage() {//final language computation if (l == null) return this.get("language");//from TLD return l; } - + public String getType() { String t = this.get("dc:type"); if (t == null) return ""; return t; } - + public String getFormat() { String t = this.get("dc:format"); if (t == null) return ""; return t; } - + public String getSource() { String t = this.get("dc:source"); if (t == null) return ""; return t; } - + public String getRights() { String t = this.get("dc:rights"); if (t == null) return ""; return t; } - + public String getTitle() { String t = this.get("title"); if (t == null) t = this.get("dc:title"); @@ -204,14 +204,14 @@ public String getTitle() { if (t == null) return ""; return t; } - + public String getPublisher() { String t = this.get("dc:publisher"); t = stripCDATA(t); if (t == null) return ""; return t; } - + public String getCreator() { String t = this.get("author"); if (t == null) t = this.get("dc:creator"); @@ -219,7 +219,7 @@ public String getCreator() { if (t == null) return ""; return t; } - + public String getDescription() { String t = this.get("body"); if (t == null) t = this.get("dc:description"); @@ -227,7 +227,7 @@ public String getDescription() { if (t == null) return ""; return t; } - + public String[] getSubject() { String t = this.get("categories"); if (t == null) this.get("dc:subject"); @@ -235,23 +235,23 @@ public String[] getSubject() { if (t == null) return new String[]{}; return t.split(";"); } - - public float getLon() { + + public double getLon() { String t = this.get("geo:long"); if (t == null) this.get("geo:lon"); t = stripCDATA(t); - if (t == null) return 0.0f; - return Float.parseFloat(t); + if (t == null) return 0.0d; + return Double.parseDouble(t); } - - public float getLat() { + + public double getLat() { String t = this.get("geo:lat"); if (t == null) this.get("geo:lat"); t = stripCDATA(t); - if (t == null) return 0.0f; - return Float.parseFloat(t); + if (t == null) return 0.0d; + return Double.parseDouble(t); } - + private String stripCDATA(String s) { if (s == null) return null; s = s.trim(); @@ -259,11 +259,11 @@ private String stripCDATA(String s) { if (s.endsWith("]]")) s = s.substring(0, s.length() - 2); return s; } - + public Document document() { HashSet languages = new HashSet(); languages.add(getLanguage()); - + return new Document( getIdentifier(true), "text/html", @@ -283,7 +283,7 @@ public Document document() { null, false); } - + public void writeXML(OutputStreamWriter os) throws IOException { Document doc = document(); if (doc != null) { diff --git a/source/net/yacy/document/content/dao/PhpBB3Dao.java b/source/net/yacy/document/content/dao/PhpBB3Dao.java index 3c5fbbcf71..486f23962c 100644 --- a/source/net/yacy/document/content/dao/PhpBB3Dao.java +++ b/source/net/yacy/document/content/dao/PhpBB3Dao.java @@ -214,7 +214,7 @@ protected DCEntry parseResultSet(ResultSet rs) throws SQLException, MalformedURL String text = xmlCleaner(rs.getString("post_text")); String user = getUser(rs.getInt("poster_id")); Date date = new Date(rs.getLong("post_time") * 1000L); - return new DCEntry(url, date, subject, user, text, 0.0f, 0.0f); + return new DCEntry(url, date, subject, user, text, 0.0d, 0.0d); } public static String xmlCleaner(String s) { diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index 2ebe2efdf6..269980a131 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -130,7 +130,7 @@ private Tag(final TagType type) { private final List li; private final CharBuffer content; private final EventListenerList htmlFilterEventListeners; - private float lon, lat; + private double lon, lat; private MultiProtocolURI canonical; @@ -803,7 +803,7 @@ public String getRefreshPath() { // // - public float getLon() { + public double getLon() { if (this.lon != 0.0f) return this.lon; String s = this.metas.get("ICBM"); // InterContinental Ballistic Missile (abbrev. supposed to be a joke: http://www.jargon.net/jargonfile/i/ICBMaddress.html), see http://geourl.org/add.html#icbm if (s != null) { @@ -811,8 +811,8 @@ public float getLon() { if (p < 0) p = s.indexOf(','); if (p < 0) p = s.indexOf(' '); if (p > 0) { - this.lat = Float.parseFloat(s.substring(0, p).trim()); - this.lon = Float.parseFloat(s.substring(p + 1).trim()); + this.lat = Double.parseDouble(s.substring(0, p).trim()); + this.lon = Double.parseDouble(s.substring(p + 1).trim()); } } if (this.lon != 0.0f) return this.lon; @@ -822,15 +822,15 @@ public float getLon() { if (p < 0) p = s.indexOf(','); if (p < 0) p = s.indexOf(' '); if (p > 0) { - this.lat = Float.parseFloat(s.substring(0, p).trim()); - this.lon = Float.parseFloat(s.substring(p + 1).trim()); + this.lat = Double.parseDouble(s.substring(0, p).trim()); + this.lon = Double.parseDouble(s.substring(p + 1).trim()); } } return this.lon; } - public float getLat() { - if (this.lat != 0.0f) return this.lat; + public double getLat() { + if (this.lat != 0.0d) return this.lat; getLon(); // parse with getLon() method which creates also the lat value return this.lat; } diff --git a/source/net/yacy/kelondro/data/meta/URIMetadata.java b/source/net/yacy/kelondro/data/meta/URIMetadata.java index 27a0a446cf..7b771d1dba 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadata.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadata.java @@ -9,12 +9,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -31,17 +31,17 @@ public interface URIMetadata extends URIReference { public String dc_title(); - + public String dc_creator(); - + public String dc_publisher(); - + public String dc_subject(); - - public float lat(); - - public float lon(); - + + public double lat(); + + public double lon(); + public long ranking(); public Date loaddate(); @@ -71,13 +71,13 @@ public interface URIMetadata extends URIReference { public int lvideo(); public int lapp(); - + public String snippet(); public Reference word(); public boolean isOlder(final URIMetadata other); - + public String toString(final String snippet); } diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index e8d5d3be24..aefff49309 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -123,7 +123,7 @@ public URIMetadataRow( final String dc_creator, final String dc_subject, final String dc_publisher, - final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f; + final double lon, final double lat, // decimal degrees as in WGS84; if unknown both values may be 0.0d; final Date mod, final Date load, final Date fresh, @@ -194,8 +194,8 @@ public static byte[] encodeComp( final String dc_creator, final String dc_subject, final String dc_publisher, - final float lat, - final float lon) { + final double lat, + final double lon) { final CharBuffer s = new CharBuffer(3600, 360); s.append(url.toNormalform(false, true)).appendLF(); s.append(dc_title).appendLF(); @@ -205,7 +205,7 @@ public static byte[] encodeComp( s.appendLF(); if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher); s.appendLF(); - if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF(); + if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Double.toString(lat)).append(',').append(Double.toString(lon)).appendLF(); return UTF8.getBytes(s.toString()); } @@ -418,11 +418,11 @@ public String dc_subject() { return this.metadata().dc_subject(); } - public float lat() { + public double lat() { return this.metadata().lat(); } - public float lon() { + public double lon() { return this.metadata().lon(); } @@ -638,15 +638,15 @@ public DigestURI url() { public String dc_creator() { return this.dc_creator; } public String dc_publisher() { return this.dc_publisher; } public String dc_subject() { return this.dc_subject; } - public float lat() { - if (this.latlon == null || this.latlon.length() == 0) return 0.0f; + public double lat() { + if (this.latlon == null || this.latlon.length() == 0) return 0.0d; final int p = this.latlon.indexOf(','); - return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(0, p)); + return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p)); } - public float lon() { - if (this.latlon == null || this.latlon.length() == 0) return 0.0f; + public double lon() { + if (this.latlon == null || this.latlon.length() == 0) return 0.0d; final int p = this.latlon.indexOf(','); - return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(p + 1)); + return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1)); } } diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index 257ace2cad..71e52bc8c9 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -377,7 +377,7 @@ public URIMetadataRow storeDocument( document.dc_subject(' '), // tags document.dc_publisher(), // publisher (may be important to get location data) document.lon(), // decimal degrees as in WGS84; - document.lat(), // if unknown both values may be 0.0f; + document.lat(), // if unknown both values may be 0.0d; modDate, // modification date loadDate, // loaded date new Date(loadDate.getTime() + Math.max(0, loadDate.getTime() - modDate.getTime()) / 2), // freshdate, computed with Proxy-TTL formula diff --git a/source/net/yacy/search/index/SolrConfiguration.java b/source/net/yacy/search/index/SolrConfiguration.java index 5855ad99e5..f898ed248d 100644 --- a/source/net/yacy/search/index/SolrConfiguration.java +++ b/source/net/yacy/search/index/SolrConfiguration.java @@ -108,6 +108,10 @@ protected void addSolr(final SolrDoc solrdoc, final SolrField key, final float v if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value); } + protected void addSolr(final SolrDoc solrdoc, final SolrField key, final double value) { + if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value); + } + protected void addSolr(final SolrDoc solrdoc, final SolrField key, final boolean value) { if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value); } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 4c313365aa..9a6f19c9e5 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -141,6 +141,7 @@ public String getModifier() { public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options public final String userAgent; public boolean filterfailurls; + public double lat, lon, radius; public QueryParams( final String queryString, @@ -198,6 +199,9 @@ public QueryParams( this.userAgent = userAgent; this.transmitcount = 0; this.filterfailurls = false; + this.lat = 0.0d; + this.lon = 0.0d; + this.radius = 0.0d; } public QueryParams( @@ -224,7 +228,8 @@ public QueryParams( final Segment indexSegment, final RankingProfile ranking, final String userAgent, - final boolean filterfailurls) { + final boolean filterfailurls, + final double lat, final double lon, final double radius) { this.queryString = queryString; this.queryHashes = queryHashes; @@ -271,8 +276,15 @@ public QueryParams( this.userAgent = userAgent; this.transmitcount = 0; this.filterfailurls = filterfailurls; + // we normalize here the location and radius because that should cause a better caching + // and as surplus it will increase privacy + this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal; + this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal; + this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal; } + double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid + public Segment getSegment() { return this.indexSegment; } @@ -511,6 +523,8 @@ public String id(final boolean anonymized) { context.append(asterisk); context.append(this.modifier.s); context.append(asterisk); + context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius); + context.append(asterisk); context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()); String result = context.toString(); if (anonymized) { diff --git a/source/net/yacy/search/query/RWIProcess.java b/source/net/yacy/search/query/RWIProcess.java index 70137333c7..372bf23ea9 100644 --- a/source/net/yacy/search/query/RWIProcess.java +++ b/source/net/yacy/search/query/RWIProcess.java @@ -629,6 +629,18 @@ public URIMetadataRow takeURL(final boolean skipDoubleDom, final long waitingtim continue; } + // check geo coordinates + double lat, lon; + if (this.query.radius > 0.0d && this.query.lat != 0.0d && this.query.lon != 0.0d && (lat = page.lat()) > 0.0d && (lon = page.lon()) > 0.0d) { + double latDelta = this.query.lat - lat; + double lonDelta = this.query.lon - lon; + double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras + if (distance > this.query.radius) { + this.sortout++; + continue; + } + } + // check vocabulary constraint final String tags = page.dc_subject(); final String[] taglist = tags == null || tags.length() == 0 ? new String[0] : SPACE_PATTERN.split(page.dc_subject()); diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index f3404054a4..600d3bba37 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -360,7 +360,7 @@ public void deployWorker(int deployCount, final int neededResults) { Worker worker; if (this.workerThreads == null) { this.workerThreads = new Worker[deployCount]; - synchronized(this.workerThreads) { + synchronized(this.workerThreads) {try { for (int i = 0; i < this.workerThreads.length; i++) { if (this.result.sizeAvailable() >= neededResults || (this.rankingProcess.feedingIsFinished() && this.rankingProcess.sizeQueue() == 0)) { @@ -376,7 +376,7 @@ public void deployWorker(int deployCount, final int neededResults) { } } } - } + } catch (OutOfMemoryError e) {}} } else { // there are still worker threads running, but some may be dead. // if we find dead workers, reanimate them diff --git a/source/net/yacy/search/snippet/ResultEntry.java b/source/net/yacy/search/snippet/ResultEntry.java index d1177f5c7d..f6adde6dc0 100644 --- a/source/net/yacy/search/snippet/ResultEntry.java +++ b/source/net/yacy/search/snippet/ResultEntry.java @@ -182,10 +182,10 @@ public int lvideo() { public int lapp() { return this.urlentry.lapp(); } - public float lat() { + public double lat() { return this.urlentry.lat(); } - public float lon() { + public double lon() { return this.urlentry.lon(); } public WordReferenceVars word() {