Merge pull request #375 from AtlasOfLivingAustralia/sitemap

grails6, alaSecurityLibsVersion, ala-bootstrap3, sitemap

.travis.yml

@@ -1,14 +1,10 @@
 language: groovy
 jdk:
 - openjdk11
-sudo: false
 branches:
   only:
   - master
   - develop
-  - hotfix
-  - 1.4.2.1
-  - feature/grails5
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock
   - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
@@ -19,6 +15,10 @@ cache:
   - $HOME/.gradle/caches/
   - $HOME/.gradle/wrapper/
 
+install:
+  - 'travis_wait 30 ./gradlew clean'
+  - './gradlew assemble'
+
 after_success:
   - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && travis_retry ./gradlew publish'
 

build.gradle

@@ -1,29 +1,25 @@
-buildscript {
-    repositories {
-        mavenLocal()
-        maven { url "https://nexus.ala.org.au/content/groups/public/" }
-        maven { url "https://repo.grails.org/grails/core" }
-    }
-    dependencies {
-        classpath "org.grails:grails-gradle-plugin:$grailsVersion"
-        classpath "gradle.plugin.com.github.erdi.webdriver-binaries:webdriver-binaries-gradle-plugin:$webdriverBinariesVersion"
-        classpath "com.bertramlabs.plugins:asset-pipeline-gradle:3.4.6"
-    }
+plugins {
+    id "groovy"
+    id "org.grails.grails-gsp"
+    id "org.grails.grails-web"
+    id "com.github.erdi.webdriver-binaries" version "3.0"
+    id "war"
+    id "idea"
+    id "com.bertramlabs.asset-pipeline"
+    id "application"
+    id "eclipse"
+
+    id "com.gorylenko.gradle-git-properties" version "2.4.1"
+
+    id "maven-publish"
 }
 
-version "2.2.0-SNAPSHOT"
+version "3.0.0-SNAPSHOT"
 group "au.org.ala"
 
-apply plugin:"eclipse"
-apply plugin:"idea"
-apply plugin:"war"
-apply plugin:"org.grails.grails-web"
-apply plugin:"com.github.erdi.webdriver-binaries"
-apply plugin:"com.bertramlabs.asset-pipeline"
-apply plugin:"org.grails.grails-gsp"
-apply plugin:"maven-publish"
 
 publishing {
+    targetCompatibility = 1.11
     repositories {
         maven {
             name 'Nexus'
@@ -43,11 +39,22 @@ publishing {
 
 bootWar {
     launchScript()
+    dependsOn(compileGroovyPages)
 }
 
+war {
+    dependsOn(compileGroovyPages)
+}
+
+java {
+    sourceCompatibility = JavaVersion.toVersion("11")
+}
+
+
 repositories {
     mavenLocal()
     maven { url "https://nexus.ala.org.au/content/groups/public/" }
+    mavenCentral()
     maven { url "https://repo.grails.org/grails/core" }
 }
 configurations {
@@ -88,11 +95,11 @@ dependencies {
     testImplementation  "org.mockito:mockito-core"
     testImplementation  "org.grails:grails-web-testing-support"
     testImplementation  "org.grails.plugins:geb"
-    testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion"
-    testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion"
-    testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion"
-    testImplementation "org.seleniumhq.selenium:selenium-chrome-driver:$seleniumVersion"
-    testImplementation "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion"
+//    testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion"
+//    testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion"
+//    testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion"
+//    testImplementation "org.seleniumhq.selenium:selenium-chrome-driver:$seleniumVersion"
+//    testImplementation "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion"
     testImplementation  'org.springframework:spring-messaging:5.3.22'
 
     implementation 'com.github.davidmoten:rxjava-file:0.4'
@@ -111,7 +118,7 @@ dependencies {
     }
     implementation "com.google.guava:guava:19.0"
     implementation 'dk.glasius:external-config:3.1.1'
-    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.3.0"
+    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.4.0"
     implementation "org.grails.plugins:ala-ws-security-plugin:${alaSecurityLibsVersion}"
     implementation "org.grails.plugins:ala-ws-plugin:${alaSecurityLibsVersion}"
     implementation "org.grails.plugins:ala-auth:${alaSecurityLibsVersion}"
@@ -142,17 +149,17 @@ tasks.withType(GroovyCompile) {
     }
 }
 
-webdriverBinaries {
-    chromedriver "$chromeDriverVersion"
-    geckodriver "$geckodriverVersion"
-}
-
-tasks.withType(Test) {
-    systemProperty "geb.env", System.getProperty('geb.env')
-    systemProperty "geb.build.reportsDir", reporting.file("geb/integrationTest")
-    systemProperty "webdriver.chrome.driver", System.getProperty('webdriver.chrome.driver')
-    systemProperty "webdriver.gecko.driver", System.getProperty('webdriver.gecko.driver')
-}
+//webdriverBinaries {
+//    chromedriver "$chromeDriverVersion"
+//    geckodriver "$geckodriverVersion"
+//}
+
+//tasks.withType(Test) {
+//    systemProperty "geb.env", System.getProperty('geb.env')
+//    systemProperty "geb.build.reportsDir", reporting.file("geb/integrationTest")
+//    systemProperty "webdriver.chrome.driver", System.getProperty('webdriver.chrome.driver')
+//    systemProperty "webdriver.gecko.driver", System.getProperty('webdriver.gecko.driver')
+//}
 
 
 assets {

gradle.properties

@@ -1,5 +1,7 @@
-grailsVersion=5.2.1
+grailsVersion=6.0.0
+grailsGradlePluginVersion=6.0.0
 gormVersion=7.2.1
+org.gradle.caching=true
 org.gradle.daemon=true
 org.gradle.parallel=true
 org.gradle.jvmargs=-Dfile.encoding=UTF-8 -Xmx1024M
@@ -10,4 +12,4 @@ webdriverBinariesVersion=2.6
 chromeDriverVersion=2.45.0
 geckodriverVersion=0.24.0
 seleniumSafariDriverVersion=3.14.0
-alaSecurityLibsVersion=6.1.0
+alaSecurityLibsVersion=6.2.0

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,10 +80,10 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
@@ -143,12 +143,16 @@ fi
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045 
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045 
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -205,6 +209,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,8 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

grails-app/conf/application.yml

@@ -273,9 +273,9 @@ skin:
   orgNameLong: Atlas of Living Australia
 useLegacyAuto: false
 import:
-  sequence: collectory,taxonomy-all,vernacular,denormalise,layers,regions,localities,conservation-lists,wordpress,knowledgebase,favourites,weights,link-identifiers,images,occurrences,hidden-images,wiki-urls,suggest-index,swap
-  sequenceDaily: conservation-lists,wordpress,knowledgebase,favourites,suggest-index,swap
-  sequenceWeekly: images,hidden-images,wiki-urls,occurrences,layers,regions,localities,suggest-index,swap
+  sequence: collectory,taxonomy-all,vernacular,denormalise,layers,regions,localities,conservation-lists,wordpress,knowledgebase,favourites,weights,link-identifiers,images,occurrences,hidden-images,wiki-urls,suggest-index,sitemap,swap
+  sequenceDaily: conservation-lists,wordpress,knowledgebase,favourites,suggest-index,images,hidden-images,wiki-urls,sitemap,swap
+  sequenceWeekly: occurrences,layers,regions,localities,suggest-index,sitemap,swap
   # enable daily and weekly tasks
   enableTasks: false
   # run daily sequence on this hour
@@ -300,3 +300,7 @@ import:
 headerAndFooter:
   baseURL: "https://www.ala.org.au/commonui-bs3-2019"
   version: "2"
+
+sitemap:
+  dir: /data/bie-index
+  enabled: true

grails-app/controllers/au/org/ala/bie/ImportController.groovy

@@ -60,6 +60,8 @@ class ImportController {
 
     def links(){}
 
+    def sitemap() {}
+
     def occurrences(){}
 
     def swap() {
@@ -235,6 +237,17 @@ class ImportController {
         asJson(job.status())
     }
 
+    /**
+     * Build sitemap.xml from SOLR index
+     *
+     * @return
+     */
+    def buildSitemap(){
+        boolean online = params.getBoolean('online', false)
+        def job = execute("buildSitemap", "admin.button.buildsitemap", { importService.buildSitemap(online) })
+        asJson(job.status())
+    }
+
     /**
      * Index occurrence data
      *

grails-app/controllers/au/org/ala/bie/SearchController.groovy

@@ -868,9 +868,10 @@ class SearchController implements GrailsConfigurationAware {
     private regularise(String guid) {
         if (!guid)
             return guid
+        guid = URLDecoder.decode(guid)
         if (guid ==~ BROKEN_URLPATTERN) {
             guid = guid.replaceFirst(":/", "://")
         }
         return guid
     }
-}
+}

grails-app/controllers/au/org/ala/bie/SitemapController.groovy

@@ -0,0 +1,31 @@
+package au.org.ala.bie
+
+class SitemapController {
+    def index(Integer idx) {
+        if (!grailsApplication.config.sitemap.enabled) {
+            response.status = 404
+            return
+        }
+
+        File index = new File(grailsApplication.config.sitemap.dir + '/sitemap.xml')
+        if (!index.exists()) {
+            response.status = 404
+            return
+        }
+
+        response.contentType = "application/xml"
+
+        if (idx == null) {
+            // return sitemap index
+            response.outputStream << index.newInputStream()
+        } else {
+            // return sitemap urls
+            File part = new File(grailsApplication.config.sitemap.dir + '/sitemap' + idx + ".xml")
+            if (!part.exists()) {
+                response.status = 404
+                return
+            }
+            response.outputStream << part.newInputStream()
+        }
+    }
+}

grails-app/controllers/bie/index/UrlMappings.groovy

@@ -61,6 +61,8 @@ class UrlMappings {
         "/api/setImages" (controller: 'misc', action: 'setImages')
         "/api/setUrl" (controller: 'misc', action: 'setWikiUrl')
 
+        "/sitemap($idx)?.xml"(controller: "sitemap", action: "index")
+
         "/"(view: "/index")
 
         "500"(view: '/error')

grails-app/i18n/messages.properties

@@ -111,6 +111,7 @@ admin.button.jobStatus.detail=Show job information
 admin.button.loadoccurrence=Add occurrence information
 admin.button.loadwordpress=Import WordPress pages
 admin.button.loadknowledgebase=Import Knowledge Base pages
+admin.button.buildsitemap=Build sitemap from SOLR index
 admin.button.loadimagesall=Load All Images
 admin.button.loadimagespref=Load Preferred Images
 admin.button.reconnect=Reconnect
@@ -148,6 +149,8 @@ admin.import.wordpress.label=Import Wordpress Pages
 admin.import.wordpress.lead=Import/Reload WordPress pages into the main search index.
 admin.import.knowledgebase.label=Import Knowledge Base Pages
 admin.import.knowledgebase.lead=Import/Reload Knowledge Base pages into the main search index.
+admin.import.sitemap.label=Build sitemap
+admin.import.sitemap.lead=Build sitemap from the SOLR index.
 admin.import.swap=Note SOLR cores (bie / bie-offline) require swapping before searches will appear.
 admin.index.title=Admin | BIE Web services | {0}
 admin.job.label=View Jobs