Geospatial search

conf/solr/8.11.1/schema.xml

@@ -228,6 +228,11 @@
 
     <field name="dsPersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="filePersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <!-- Dataverse geospatial search -->
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#rpt -->
+    <field name="geolocation" type="location_rpt" multiValued="true" stored="true" indexed="true"/>
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#bboxfield -->
+    <field name="boundingBox" type="bbox" multiValued="true" stored="true" indexed="true"/>
 
     <!--
         METADATA SCHEMA FIELDS
@@ -1104,6 +1109,9 @@
     -->
     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
                geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
+    <!-- Dataverse - per GeoBlacklight, adding field type for bboxField that enables, among other things, overlap ratio calculations -->
+    <fieldType name="bbox" class="solr.BBoxField"
+           geo="true" distanceUnits="kilometers" numberType="pdouble" />
 
     <!-- Payloaded field types -->
     <fieldType name="delimited_payloads_float" stored="false" indexed="true" class="solr.TextField">

doc/release-notes/8239-geospatial-indexing.md

@@ -0,0 +1,5 @@
+Support for indexing the "Geographic Bounding Box" fields ("West Longitude", "East Longitude", "North Latitude", and "South Latitude") from the Geospatial metadata block has been added.
+
+Geospatial search is supported but only via API using two new parameters: `geo_point` and `geo_radius`.
+
+A Solr schema update is required.

doc/sphinx-guides/source/_static/api/ddi_dataset.xml

@@ -88,12 +88,12 @@
         <geoBndBox>
           <westBL>10</westBL>
           <eastBL>20</eastBL>
-          <northBL>30</northBL>
-          <southBL>40</southBL>
+          <northBL>40</northBL>
+          <southBL>30</southBL>
         </geoBndBox>
         <geoBndBox>
-          <southBL>80</southBL>
-          <northBL>70</northBL>
+          <southBL>70</southBL>
+          <northBL>80</northBL>
           <eastBL>60</eastBL>
           <westBL>50</westBL>
         </geoBndBox>

doc/sphinx-guides/source/api/search.rst

@@ -35,6 +35,8 @@ show_relevance   boolean  Whether or not to show details of which fields were ma
 show_facets      boolean  Whether or not to show facets that can be operated on by the "fq" parameter. False by default. See :ref:`advanced search example <advancedsearch-example>`.
 fq               string   A filter query on the search term. Multiple "fq" parameters can be used. See :ref:`advanced search example <advancedsearch-example>`.
 show_entity_ids  boolean  Whether or not to show the database IDs of the search results (for developer use).
+geo_point        string	  Latitude and longitude in the form ``geo_point=42.3,-71.1``. You must supply ``geo_radius`` as well. See also :ref:`geospatial-search`.
+geo_radius       string	  Radial distance in kilometers from ``geo_point`` (which must be supplied as well) such as ``geo_radius=1.5``.
 metadata_fields  string	  Includes the requested fields for each dataset in the response. Multiple "metadata_fields" parameters can be used to include several fields. The value must be in the form "{metadata_block_name}:{field_name}" to include a specific field from a metadata block (see :ref:`example <dynamic-citation-some>`) or "{metadata_field_set_name}:\*" to include all the fields for a metadata block (see :ref:`example <dynamic-citation-all>`). "{field_name}" cannot be a subfield of a compound field. If "{field_name}" is a compound field, all subfields are included.
 ===============  =======  ===========
 

doc/sphinx-guides/source/user/find-use-data.rst

@@ -39,6 +39,13 @@ enter search terms for Dataverse collections, dataset metadata (citation and dom
 metadata. If you are searching for tabular data files you can also search at the variable level for name and label. To find 
 out more about what each field searches, hover over the field name for a detailed description of the field.
 
+.. _geospatial-search:
+
+Geospatial Search
+-----------------
+
+Geospatial search is available from the :doc:`/api/search` (look for "geo" parameters). The metadata fields that are geospatially indexed are "West Longitude", "East Longitude", "North Latitude", and "South Latitude" from the "Geographic Bounding Box" field in the "Geospatial Metadata" block.
+
 Browsing a Dataverse Installation
 ---------------------------------
 

src/main/java/edu/harvard/iq/dataverse/api/Search.java

@@ -72,6 +72,8 @@ public Response search(
             @QueryParam("show_my_data") boolean showMyData,
             @QueryParam("query_entities") boolean queryEntities,
             @QueryParam("metadata_fields") List<String> metadataFields,
+            @QueryParam("geo_point") String geoPointRequested,
+            @QueryParam("geo_radius") String geoRadiusRequested,
             @Context HttpServletResponse response
     ) {
 
@@ -87,6 +89,8 @@ public Response search(
             // sanity checking on user-supplied arguments
             SortBy sortBy;
             int numResultsPerPage;
+            String geoPoint;
+            String geoRadius;
             List<Dataverse> dataverseSubtrees = new ArrayList<>();
 
             try {
@@ -119,6 +123,17 @@ public Response search(
                     throw new IOException("Filter is empty, which should never happen, as this allows unfettered searching of our index");
                 }
 
+                geoPoint = getGeoPoint(geoPointRequested);
+                geoRadius = getGeoRadius(geoRadiusRequested);
+
+                if (geoPoint != null && geoRadius == null) {
+                    return error(Response.Status.BAD_REQUEST, "If you supply geo_point you must also supply geo_radius.");
+                }
+
+                if (geoRadius != null && geoPoint == null) {
+                    return error(Response.Status.BAD_REQUEST, "If you supply geo_radius you must also supply geo_point.");
+                }
+
             } catch (Exception ex) {
                 return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage());
             }
@@ -137,7 +152,9 @@ public Response search(
                         paginationStart,
                         dataRelatedToMe,
                         numResultsPerPage,
-                        true //SEK get query entities always for search API additional Dataset Information 6300  12/6/2019
+                        true, //SEK get query entities always for search API additional Dataset Information 6300  12/6/2019
+                        geoPoint,
+                        geoRadius
                 );
             } catch (SearchException ex) {
                 Throwable cause = ex;
@@ -340,4 +357,12 @@ private Dataverse getSubtree(String alias) throws Exception {
         }
     }
 
+    private String getGeoPoint(String geoPointRequested) {
+        return SearchUtil.getGeoPoint(geoPointRequested);
+    }
+
+    private String getGeoRadius(String geoRadiusRequested) {
+        return SearchUtil.getGeoRadius(geoRadiusRequested);
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -6,6 +6,7 @@
 import edu.harvard.iq.dataverse.DataFileTag;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
+import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
 import edu.harvard.iq.dataverse.DatasetFieldConstant;
 import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
 import edu.harvard.iq.dataverse.DatasetFieldType;
@@ -37,6 +38,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.sql.Timestamp;
+import java.text.NumberFormat;
 import java.text.SimpleDateFormat;
 import java.time.LocalDate;
 import java.util.ArrayList;
@@ -947,6 +949,70 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                         }
                     }
                 }
+
+                //ToDo - define a geom/bbox type solr field and find those instead of just this one
+                if(dsfType.getName().equals(DatasetFieldConstant.geographicBoundingBox)) {
+                    String minWestLon=null;
+                    String maxEastLon=null;
+                    String maxNorthLat=null;
+                    String minSouthLat=null;
+                    for (DatasetFieldCompoundValue compoundValue : dsf.getDatasetFieldCompoundValues()) {
+                        String westLon=null;
+                        String eastLon=null;
+                        String northLat=null;
+                        String southLat=null;
+                        for(DatasetField childDsf: compoundValue.getChildDatasetFields()) {
+                            switch (childDsf.getDatasetFieldType().getName()) {
+                            case DatasetFieldConstant.westLongitude:
+                                westLon = childDsf.getRawValue();
+                                break;
+                            case DatasetFieldConstant.eastLongitude:
+                                eastLon = childDsf.getRawValue();
+                                break;
+                            case DatasetFieldConstant.northLatitude:
+                                northLat = childDsf.getRawValue();
+                                break;
+                            case DatasetFieldConstant.southLatitude:
+                                southLat = childDsf.getRawValue();
+                                break;
+                            }
+                        }
+                        if ((eastLon != null || westLon != null) && (northLat != null || southLat != null)) {
+                            // we have a point or a box, so proceed
+                            if (eastLon == null) {
+                                eastLon = westLon;
+                            } else if (westLon == null) {
+                                westLon = eastLon;
+                            }
+                            if (northLat == null) {
+                                northLat = southLat;
+                            } else if (southLat == null) {
+                                southLat = northLat;
+                            }
+                            //Find the overall bounding box that includes all bounding boxes
+                            if(minWestLon==null || Float.parseFloat(minWestLon) > Float.parseFloat(westLon)) {
+                                minWestLon=westLon;
+                            }
+                            if(maxEastLon==null || Float.parseFloat(maxEastLon) < Float.parseFloat(eastLon)) {
+                                maxEastLon=eastLon;
+                            }
+                            if(minSouthLat==null || Float.parseFloat(minSouthLat) > Float.parseFloat(southLat)) {
+                                minSouthLat=southLat;
+                            }
+                            if(maxNorthLat==null || Float.parseFloat(maxNorthLat) < Float.parseFloat(northLat)) {
+                                maxNorthLat=northLat;
+                            }
+                            //W, E, N, S
+                            solrInputDocument.addField(SearchFields.GEOLOCATION, "ENVELOPE(" + westLon + "," + eastLon + "," + northLat + "," + southLat + ")");
+                        }
+                    }
+                    //Only one bbox per dataset
+                    //W, E, N, S
+                    if ((minWestLon != null || maxEastLon != null) && (maxNorthLat != null || minSouthLat != null)) {
+                        solrInputDocument.addField(SearchFields.BOUNDING_BOX, "ENVELOPE(" + minWestLon + "," + maxEastLon + "," + maxNorthLat + "," + minSouthLat + ")");
+                    }
+
+                }
             }
 
             for(String metadataBlockName : metadataBlocksWithValue) {

src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java

@@ -268,4 +268,9 @@ more targeted results for just datasets. The format is YYYY (i.e.
     public static final String FULL_TEXT = "_text_";
     public static final String EMBARGO_END_DATE = "embargoEndDate";
 
+    // SpatialRecursivePrefixTreeFieldType: https://solr.apache.org/guide/8_11/spatial-search.html#rpt
+    public static final String GEOLOCATION = "geolocation";
+    // BBoxField (bounding box): https://solr.apache.org/guide/8_11/spatial-search.html#bboxfield
+    public static final String BOUNDING_BOX = "boundingBox";
+
 }

src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java

@@ -355,15 +355,15 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest);
             List<Dataverse> dataverses = new ArrayList<>();
             dataverses.add(dataverse);
-            solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false);
+            solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
             if (solrQueryResponse.hasError()){
                 logger.info(solrQueryResponse.getError());
                 setSolrErrorEncountered(true);
             }
             // This 2nd search() is for populating the "type" ("dataverse", "dataset", "file") facets: -- L.A. 
             // (why exactly do we need it, again?)
             // To get the counts we display in the types facets particulary for unselected types - SEK 08/25/2021
-            solrQueryResponseAllTypes = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalAllTypes, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false);
+            solrQueryResponseAllTypes = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalAllTypes, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
             if (solrQueryResponse.hasError()){
                 logger.info(solrQueryResponse.getError());
                 setSolrErrorEncountered(true);

src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java

@@ -100,7 +100,7 @@ public class SearchServiceBean {
      * @throws SearchException
      */
     public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Dataverse> dataverses, String query, List<String> filterQueries, String sortField, String sortOrder, int paginationStart, boolean onlyDatatRelatedToMe, int numResultsPerPage) throws SearchException {
-        return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true);
+        return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true, null, null);
     }
 
     /**
@@ -121,10 +121,24 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Datavers
      * @param onlyDatatRelatedToMe
      * @param numResultsPerPage
      * @param retrieveEntities - look up dvobject entities with .find() (potentially expensive!)
+     * @param geoPoint e.g. "35,15"
+     * @param geoRadius e.g. "5"
      * @return
      * @throws SearchException
      */
-    public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Dataverse> dataverses, String query, List<String> filterQueries, String sortField, String sortOrder, int paginationStart, boolean onlyDatatRelatedToMe, int numResultsPerPage, boolean retrieveEntities) throws SearchException {
+    public SolrQueryResponse search(
+            DataverseRequest dataverseRequest,
+            List<Dataverse> dataverses,
+            String query,
+            List<String> filterQueries,
+            String sortField, String sortOrder,
+            int paginationStart,
+            boolean onlyDatatRelatedToMe,
+            int numResultsPerPage,
+            boolean retrieveEntities,
+            String geoPoint,
+            String geoRadius
+    ) throws SearchException {
 
         if (paginationStart < 0) {
             throw new IllegalArgumentException("paginationStart must be 0 or greater");
@@ -204,8 +218,12 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Datavers
         for (String filterQuery : filterQueries) {
             solrQuery.addFilterQuery(filterQuery);
         }
-
-
+        if (geoPoint != null && !geoPoint.isBlank() && geoRadius != null && !geoRadius.isBlank()) {
+            solrQuery.setParam("pt", geoPoint);
+            solrQuery.setParam("d", geoRadius);
+            // See https://solr.apache.org/guide/8_11/spatial-search.html#bbox
+            solrQuery.addFilterQuery("{!bbox sfield=" + SearchFields.GEOLOCATION + "}");
+        }
 
         // -----------------------------------
         // Facets to Retrieve

src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java

@@ -181,5 +181,49 @@ public static String constructQuery(List<String> queryStrings, boolean isAnd, bo
 
         return queryBuilder.toString().trim();
     }
-
+
+    /**
+     * @return Null if supplied point is null or whitespace.
+     * @throws IllegalArgumentException If the lat/long is not separated by a
+     * comma.
+     * @throws NumberFormatException If the lat/long values are not numbers.
+     */
+    public static String getGeoPoint(String userSuppliedGeoPoint) throws IllegalArgumentException, NumberFormatException {
+        if (userSuppliedGeoPoint == null || userSuppliedGeoPoint.isBlank()) {
+            return null;
+        }
+        String[] parts = userSuppliedGeoPoint.split(",");
+        // We'll supply our own errors but Solr gives a decent one:
+        // "Point must be in 'lat, lon' or 'x y' format: 42.3;-71.1"
+        if (parts.length != 2) {
+            String msg = "Must contain a single comma to separate latitude and longitude.";
+            throw new IllegalArgumentException(msg);
+        }
+        float latitude = Float.parseFloat(parts[0]);
+        float longitude = Float.parseFloat(parts[1]);
+        return latitude + "," + longitude;
+    }
+
+    /**
+     * @return Null if supplied radius is null or whitespace.
+     * @throws NumberFormatException If the radius is not a positive number.
+     */
+    public static String getGeoRadius(String userSuppliedGeoRadius) throws NumberFormatException {
+        if (userSuppliedGeoRadius == null || userSuppliedGeoRadius.isBlank()) {
+            return null;
+        }
+        float radius = 0;
+        try {
+            radius = Float.parseFloat(userSuppliedGeoRadius);
+        } catch (NumberFormatException ex) {
+            String msg = "Non-number radius supplied.";
+            throw new NumberFormatException(msg);
+        }
+        if (radius <= 0) {
+            String msg = "The supplied radius must be greater than zero.";
+            throw new NumberFormatException(msg);
+        }
+        return userSuppliedGeoRadius;
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java

@@ -266,7 +266,9 @@ private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchExcepti
                 paginationStart,
                 dataRelatedToMe,
                 numResultsPerPage,
-                false // do not retrieve entities
+                false, // do not retrieve entities
+                null,
+                null
         );
         return solrQueryResponse;
     }