Merge remote-tracking branch 'upstream/master'

build.gradle

@@ -40,7 +40,7 @@ repositories {
 }
 
 group = 'au.org.ala'
-version = '3.3.1'
+version = '3.4.1'
 
 
 boolean inplace = false
@@ -78,7 +78,7 @@ dependencies {
     implementation 'org.geotools:gt-opengis:26.0'
     implementation 'org.geotools:gt-referencing:26.0'
     implementation 'org.geotools:gt-epsg-wkt:26.0'
-    implementation 'au.org.ala.names:ala-namematching-client:1.4'
+    implementation 'au.org.ala.names:ala-namematching-client:1.8.1'
     implementation 'com.googlecode.json-simple:json-simple:1.1.1'
     implementation 'au.org.ala:ala-logger:1.4.1'
     implementation 'au.org.ala:ala-common-ui:1.0'
@@ -118,7 +118,8 @@ dependencies {
     implementation 'au.org.ala:data-quality-filter-service-client:1.0.0'
     implementation 'com.auth0:java-jwt:3.18.2'
     implementation 'com.auth0:jwks-rsa:0.20.0'
-    implementation 'org.gbif:gbif-api:0.157'
+    implementation 'org.gbif:gbif-api:1.12.11'
+    implementation 'org.gbif:dwc-api:1.47'
     implementation 'org.apache.commons:commons-compress:1.21'
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
     implementation 'org.springframework.boot:spring-boot-autoconfigure:2.5.7'

src/main/java/au/org/ala/biocache/config/AppConfig.java

@@ -20,6 +20,8 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.support.AbstractMessageSource;
 import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.security.web.firewall.HttpFirewall;
+import org.springframework.security.web.firewall.StrictHttpFirewall;
 import org.springframework.web.servlet.ViewResolver;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@@ -130,4 +132,12 @@ public void addViewControllers(ViewControllerRegistry registry) {
     public FqConverter fqConverter() {
         return new FqConverter();
     }
+
+    @Bean
+    public HttpFirewall configureFirewall() {
+        // Permit LSID path variables like https://biodiversity.org.au/afd/taxa/55213c39-1809-442e-b5fb-03fb99e8d97a
+        StrictHttpFirewall strictHttpFirewall = new StrictHttpFirewall();
+        strictHttpFirewall.setAllowUrlEncodedDoubleSlash(true);
+        return strictHttpFirewall;
+    }
 }

src/main/java/au/org/ala/biocache/dao/JsonPersistentQueueDAOImpl.java

@@ -16,8 +16,11 @@
 
 import au.org.ala.biocache.dto.DownloadDetailsDTO;
 import au.org.ala.biocache.dto.DownloadRequestDTO;
+import au.org.ala.biocache.util.AlaUserProfileDeserializer;
+import au.org.ala.ws.security.profile.AlaUserProfile;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Value;
@@ -59,6 +62,11 @@ public class JsonPersistentQueueDAOImpl implements PersistentQueueDAO {
     public void init() {
         jsonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
+        // add deserializer for AlaUserProfile
+        SimpleModule module = new SimpleModule();
+        module.addDeserializer(AlaUserProfile.class, new AlaUserProfileDeserializer());
+        jsonMapper.registerModule(module);
+
         File file = new File(cacheDirectory);
         try {
             FileUtils.forceMkdir(file);
@@ -102,6 +110,9 @@ public boolean remove(DownloadDetailsDTO download) {
         logger.debug("Removing the download from the queue");
 
         try {
+            // delete it from memory
+            offlineDownloadList.remove(download);
+
             // delete it from the directory
             File f = getFile(download.getStartTime());
             if (f.exists()) {
@@ -110,9 +121,6 @@ public boolean remove(DownloadDetailsDTO download) {
                 }
                 FileUtils.deleteQuietly(f);
 
-                // delete it from memory
-                offlineDownloadList.remove(download);
-
                 return true;
             }
         } catch (Exception e) {

src/main/java/au/org/ala/biocache/dao/QidCacheDAO.java

@@ -75,5 +75,7 @@ public interface QidCacheDAO {
 
     String generateQid(SpatialSearchRequestDTO requestParams, String bbox, String title, Long maxage, String source);
 
+    String fixWkt(String wkt) throws Exception;
+
     void clear();
 }

src/main/java/au/org/ala/biocache/dao/QidCacheDAOImpl.java

@@ -21,6 +21,7 @@
 import au.org.ala.biocache.util.QidSizeException;
 import au.org.ala.biocache.util.QueryFormatUtils;
 import au.org.ala.biocache.util.SpatialUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cache.annotation.Cacheable;
@@ -70,24 +71,6 @@ public class QidCacheDAOImpl implements QidCacheDAO {
     @Value("${qid.wkt.maxPoints:5000}")
     private int maxWktPoints;
 
-    /**
-     * The simplification factor used to iteratively reduce WKT complexity to reduce index query time for qids between the initial and maximum precisions
-     */
-    @Value("${qid.wkt.simplification.factor:2.0}")
-    private double wktSimplificationFactor;
-
-    /**
-     * The initial distance precision value to attempt when reducing WKT complexity to reduce index query time for qids.
-     */
-    @Value("${qid.wkt.simplification.initialprecision:0.0001}")
-    private double wktSimplificationInitialPrecision;
-
-    /**
-     * The maximum distance precision value before giving up on iteratively reducing WKT complexity to reduce index query time for qids.
-     */
-    @Value("${qid.wkt.simplification.maxprecision:10.0}")
-    private double wktSimplificationMaxPrecision;
-
     @Inject
     private DataQualityService dataQualityService;
 
@@ -431,19 +414,7 @@ public String[] getFq(SpatialSearchRequestDTO requestParams) {
     public String generateQid(SpatialSearchRequestDTO requestParams, String bbox, String title, Long maxage, String source) {
         try {
             //simplify wkt
-            String wkt = requestParams.getWkt();
-            if (wkt != null && wkt.length() > 0) {
-                //TODO: Is this too slow? Do not want to send large WKT to SOLR.
-                wkt = fixWkt(wkt);
-
-                if (wkt == null) {
-                    //wkt too large and simplification failed, do not produce qid
-                    return null;
-                }
-
-                //set wkt
-                requestParams.setWkt(wkt);
-            }
+            requestParams.setWkt(fixWkt(requestParams.getWkt()));
 
             //get bbox (also cleans up Q)
             double[] bb = null;
@@ -479,8 +450,12 @@ public String generateQid(SpatialSearchRequestDTO requestParams, String bbox, St
     }
 
     @Cacheable("fixWkt")
-    public String fixWkt(String wkt) {
-        return SpatialUtils.simplifyWkt(wkt, maxWktPoints, wktSimplificationFactor, wktSimplificationInitialPrecision, wktSimplificationMaxPrecision);
+    @Override
+    public String fixWkt(String wkt) throws Exception {
+        if (StringUtils.isNotEmpty(wkt)) {
+            return SpatialUtils.simplifyWkt(wkt, maxWktPoints);
+        }
+        return wkt;
     }
 
     /**

src/main/java/au/org/ala/biocache/dao/SearchDAO.java

@@ -373,4 +373,6 @@ HeatmapDTO getHeatMap(String query, String[] filterQueries, Double minx, Double
     int streamingQuery(SpatialSearchRequestDTO request, ProcessInterface procSearch, ProcessInterface procFacet) throws Exception;
 
     SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean substituteDefaultFacetOrder, Map<String, String[]> extraSolrParams) throws QidMissingException;
+
+    boolean isInitialized() throws InterruptedException;
 }

src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java

@@ -271,6 +271,8 @@ public class SearchDAOImpl implements SearchDAO {
     @Value("${solr.collection:biocache1}")
     protected String solrCollection;
 
+    public CountDownLatch countDownLatch = new CountDownLatch(1);
+
     /**
      * Initialise the SOLR server instance
      */
@@ -295,6 +297,14 @@ public void init() throws Exception {
         getMaxBooleanClauses();
 
         initSensitiveFieldMapping();
+
+        countDownLatch.countDown();
+    }
+
+    @Override
+    public boolean isInitialized() throws InterruptedException {
+        countDownLatch.await();
+        return true;
     }
 
     public void refreshCaches() {
@@ -1321,16 +1331,6 @@ private String getUpperRange(String lower, Number gap, boolean addGap) {
         }
     }
 
-    protected void initDecadeBasedFacet(SolrQuery solrQuery, String field) {
-        //Solr 6.x don't use facet.date but facet.range instead
-        solrQuery.add("facet.range", field);
-        solrQuery.add("facet.range.start", DECADE_FACET_START_DATE); // facet date range starts from 1850
-        solrQuery.add("facet.range.end", "NOW/DAY"); // facet date range ends for current date (gap period)
-        solrQuery.add("facet.range.gap", "+10YEAR"); // gap interval of 10 years
-        solrQuery.add("facet.range.other", DECADE_PRE_1850_LABEL); // include counts before the facet start date ("before" label)
-        solrQuery.add("facet.range.include", "lower"); // counts will be included for dates on the starting date but not ending date
-    }
-
     /**
      * Helper method to create SolrQuery object and add facet settings
      *
@@ -1339,8 +1339,6 @@ protected void initDecadeBasedFacet(SolrQuery solrQuery, String field) {
     public SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean substituteDefaultFacetOrder, Map<String, String[]> extraSolrParams) throws QidMissingException {
         queryFormatUtils.formatSearchQuery(searchParams);
 
-        String occurrenceDate = OccurrenceIndex.OCCURRENCE_DATE;
-
         SolrQuery solrQuery = new SolrQuery();
         solrQuery.setQuery(searchParams.getFormattedQuery());
 
@@ -1358,11 +1356,7 @@ public SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean sub
         solrQuery.setFacet(searchParams.getFacet());
         if (searchParams.getFacet()) {
             for (String facet : searchParams.getFacets()) {
-                if (facet.equals(DECADE_FACET_NAME) || facet.equals("date")) {
-                    String fname = facet.equals(DECADE_FACET_NAME) ? OCCURRENCE_YEAR_INDEX_FIELD : occurrenceDate;
-                    initDecadeBasedFacet(solrQuery, fname);
-                    rangeAdded = true; // the initDecadeBasedFacet adds a range
-                } else if (facet.equals("uncertainty")) {
+                if (facet.equals("uncertainty")) {
                     Map<String, String> rangeMap = rangeBasedFacets.getRangeMap("uncertainty");
                     for (String range : rangeMap.keySet()) {
                         solrQuery.add("facet.query", range);
@@ -2022,22 +2016,22 @@ public QueryResponse searchGroupedFacets(SpatialSearchRequestDTO searchParams) t
      * @return
      */
     String getFormattedFqQuery(String facet, String value) {
-        if (facet.equals(OCCURRENCE_YEAR_INDEX_FIELD)) {
-
-            if (value.equals(DECADE_PRE_1850_LABEL)) {
-                return facet + ":" + "[* TO " + DECADE_FACET_START_DATE + "]";
-            } else {
-                SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
-                try {
-                    Date date = sdf.parse(value);
-                    Date endDate = DateUtils.addYears(date, 10);
-                    endDate = DateUtils.addMilliseconds(endDate, -1);
-                    return facet + ":" + "[" + value + " TO " + sdf.format(endDate) + "]";
-                } catch (ParseException e) {
-                    //do nothing
-                }
-            }
-        }
+//        if (facet.equals(DECADE_FACET_NAME)) {
+//
+//            if (value.equals(DECADE_PRE_1850_LABEL)) {
+//                return facet + ":" + "[* TO " + DECADE_FACET_START_DATE + "]";
+//            } else {
+//                SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
+//                try {
+//                    Date date = sdf.parse(value);
+//                    Date endDate = DateUtils.addYears(date, 10);
+//                    endDate = DateUtils.addMilliseconds(endDate, -1);
+//                    return facet + ":" + "[" + value + " TO " + sdf.format(endDate) + "]";
+//                } catch (ParseException e) {
+//                    //do nothing
+//                }
+//            }
+//        }
 
         return facet + ":\"" + value.replace("\"", "\\\"") + "\"";
     }
@@ -2057,20 +2051,6 @@ String getFacetValueDisplayName(String facet, String value) {
 
         if (facet.endsWith("_uid") || facet.endsWith("Uid")) {
             return searchUtils.getUidDisplayString(tFacet, tValue, false);
-        } else if (OccurrenceIndex.OCCURRENCE_YEAR_INDEX_FIELD.equals(facet) && value != null) {
-            try {
-                if (DECADE_PRE_1850_LABEL.equals(value)) {
-                    return messageSource.getMessage("decade.pre.start", null, "pre 1850", null); // "pre 1850";
-                }
-                SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
-                Date date = sdf.parse(value);
-                SimpleDateFormat df = new SimpleDateFormat("yyyy");
-                String year = df.format(date);
-                return year + "-" + (Integer.parseInt(year) + 9);
-            } catch (ParseException pe) {
-                return facet;
-            }
-            //1850-01-01T00:00:00Z
         } else if (searchUtils.getAuthIndexFields().contains(tFacet)) {
             //if the facet field is collector or assertion_user_id we need to perform the substitution
             Optional<AlaUserProfile> profile = authService.lookupAuthUser(value, false);

src/main/java/au/org/ala/biocache/dto/ALAOccurrenceIssue.java

@@ -213,7 +213,7 @@ private TermsGroup() {}
                             DwcTerm.genus,
                             DwcTerm.scientificName,
                             DwcTerm.scientificNameAuthorship,
-                            GbifTerm.genericName,
+                            DwcTerm.genericName,
                             DwcTerm.specificEpithet,
                             DwcTerm.infraspecificEpithet,
                             DwcTerm.taxonRank,

src/main/java/au/org/ala/biocache/dto/SpatialSearchRequestDTO.java

@@ -19,6 +19,8 @@
 import org.springframework.beans.BeanUtils;
 
 import java.util.Arrays;
+import java.util.HashSet;
+import java.util.function.Predicate;
 
 /**
  * Data Transfer Object to represent the request parameters required to perform
@@ -58,6 +60,12 @@ public static SpatialSearchRequestDTO create(SpatialSearchRequestParams params){
                 dto.setStart(0);
             }
         }
+
+        // Remove duplicate fq, remove duplicated q
+        if (dto.fq != null && dto.fq.length > 0) {
+            dto.fq = Arrays.stream(dto.fq).filter(Predicate.not(dto.q::equals)).distinct().toArray(String[]::new);
+        }
+
         return dto;
     }
 

src/main/java/au/org/ala/biocache/service/AlaImageMetadataService.java

@@ -54,23 +54,22 @@ public String getUrlFor(String imageId){
     }
 
     @Override
-    public Map<String, List<Map<String, Object>>> getImageMetadataForOccurrences(List<String> occurrenceIDs) throws Exception {
+    public Map<String, Map<String, Object>> getImageMetadataForOccurrences(List<String> imageIDs) throws Exception {
 
-        if (StringUtils.isBlank(imageServiceUrl) || occurrenceIDs == null || occurrenceIDs.isEmpty()){
-            return new HashMap<String, List<Map<String, Object>>>();
+        if (StringUtils.isBlank(imageServiceUrl) || imageIDs == null || imageIDs.isEmpty()){
+            return new HashMap<String, Map<String, Object>>();
         }
 
-        logger.debug("Retrieving the image metadata for " + occurrenceIDs.size() + " records");
+        logger.debug("Retrieving the image metadata for " + imageIDs.size() + " records");
         Map<String, Object> payload = new HashMap<String, Object>();
-        payload.put("key", "occurrenceid");
-        payload.put("values", occurrenceIDs);
+        payload.put("imageIds", imageIDs);
 
 
         HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
 
         try (CloseableHttpClient httpClient = httpClientBuilder.build()) {
 
-            HttpPost post = new HttpPost(imageServiceUrl + "/ws/findImagesByMetadata");
+            HttpPost post = new HttpPost(imageServiceUrl + "/ws/getImageInfoForIdList");
 
             ObjectMapper om = new ObjectMapper();
             post.setEntity(new StringEntity(om.writeValueAsString(payload), ContentType.APPLICATION_JSON));
@@ -83,7 +82,7 @@ public Map<String, List<Map<String, Object>>> getImageMetadataForOccurrences(Lis
                     String jsonResponseString = EntityUtils.toString(httpResponse.getEntity());
 
                     Map<String, Object> jsonResponse = om.readValue(jsonResponseString, Map.class);
-                    Map<String, List<Map<String, Object>>> imageMetadata = (Map<String, List<Map<String, Object>>>) jsonResponse.get("images");
+                    Map<String, Map<String, Object>> imageMetadata = (Map<String, Map<String, Object>>) jsonResponse.get("results");
                     logger.debug("Obtained image metadata for " + imageMetadata.size() + " records");
                     return imageMetadata;
                 }