ruODK v0.10.2: Fix #126

* odata_submission_get now explicitly only includes query parameters `skip` and `top` if non-NULL * Refreshed test fixtures for odata_submission_get * Tested locally against ODKC v1.1 and a simple test form on ODKC v1.2, but will see what GitHub Actions says (running against same ODKC v1.1 but from different OS/R versions)
ropensci · Jun 4, 2021 · 50ee03b · 50ee03b
1 parent 0f9acf8
commit 50ee03b
Show file tree

Hide file tree

Showing 17 changed files with 101 additions and 91 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Type: Package
 Package: ruODK
 Title: An R Client for the ODK Central API
-Version: 0.10.1
+Version: 0.10.2
 Authors@R: 
     c(person(given = c("Florian", "W."),
              family = "Mayer",

diff --git a/Dockerfile b/Dockerfile
@@ -1,13 +1,9 @@
-FROM rocker/geospatial:4.0.5
+FROM rocker/geospatial:4.0.5 as builder_base
 LABEL maintainer=Florian.Mayer@dbca.wa.gov.au
 LABEL description="rocker/geospatial:4.0.5 for DBCA"
-
 # Build this image with
 # docker build . -t dbcawa/ruodk:latest --build-arg GITHUB_PAT="..."
 
-ARG GITHUB_PAT
-ENV GITHUB_PAT=${GITHUB_PAT}
-
 # System dependencies ---------------------------------------------------------#
 RUN apt-get update && \
     apt remove -y libvorbis0a && \
@@ -24,6 +20,9 @@ RUN apt-get update && \
     #&& apt-get purge && #apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # R packages ------------------------------------------------------------------#
+FROM builder_base as r_libs
+ARG GITHUB_PAT
+ENV GITHUB_PAT=$GITHUB_PAT
 RUN install2.r --error \
   proj4 \
   caTools \
@@ -46,5 +45,6 @@ RUN install2.r --error \
   lattice \
   sf
 
+FROM r_libs
 RUN R -e "remotes::install_github('ropensci/ruODK@main', \
           dependencies = TRUE, ask=FALSE, update=TRUE)"
diff --git a/NEWS.md b/NEWS.md
@@ -5,6 +5,16 @@
 ## Data
 ## Maintenance
 
+# `ruODK` 0.10.2
+## Major fixes
+* Fix ODK Central v1.2 time out on NULL query parameters `skip` and `top`. 
+  ruODK now only supplies non-NULL query parameters.
+  (#126, thanks @yanokwa, @mtyszler, @thaliehln)
+## Minor fixes
+## Documentation
+## Data
+## Maintenance
+
 # `ruODK` 0.10.1
 ## Major fixes
 * `submission_export` now terminates immediately if an incorrect passphrase is 

diff --git a/R/odata_submission_get.R b/R/odata_submission_get.R
@@ -157,14 +157,12 @@ odata_submission_get <- function(table = "Submissions",
                                  filter = NULL,
                                  parse = TRUE,
                                  download = TRUE,
-                                 orders = c(
-                                   "YmdHMS",
-                                   "YmdHMSz",
-                                   "Ymd HMS",
-                                   "Ymd HMSz",
-                                   "Ymd",
-                                   "ymd"
-                                 ),
+                                 orders = c("YmdHMS",
+                                            "YmdHMSz",
+                                            "Ymd HMS",
+                                            "Ymd HMSz",
+                                            "Ymd",
+                                            "ymd"),
                                  local_dir = "media",
                                  pid = get_default_pid(),
                                  fid = get_default_fid(),
@@ -182,14 +180,21 @@ odata_submission_get <- function(table = "Submissions",
   ru_msg_info("Downloading submissions...", verbose = verbose)
 
   qry <- list(
-    `$skip` = skip %||% "",
-    `$top` = top %||% "",
     `$count` = ifelse(count == FALSE, "false", "true"),
     `$wkt` = ifelse(wkt == FALSE, "false", "true")
-    # `$filter` = ifelse(odkc_version>=1.1, filter %||% "", "")
   )
 
-  if(odkc_version>=1.1 && !is.null(filter) && filter != "") {
+  if (!is.null(skip)) {
+    qry$`$skip` <- as.integer(skip)
+    "Skipping first {skip} records" %>%
+      glue::glue() %>% ru_msg_info(verbose = verbose)
+  }
+  if (!is.null(top)) {
+    qry$`$top` <- as.integer(top)
+    "Limiting to max {top} records" %>%
+      glue::glue() %>% ru_msg_info(verbose = verbose)
+  }
+  if (odkc_version >= 1.1 && !is.null(filter) && filter != "") {
     qry$`$filter` <- as.character(filter)
   }
 
@@ -237,15 +242,12 @@ odata_submission_get <- function(table = "Submissions",
 
   # Rectangle, handle date/times, attachments, geopoints, geotraces, geoshapes
   sub <- sub %>%
-    odata_submission_rectangle(
-      form_schema = fs,
-      verbose = verbose
-    ) %>%
-    handle_ru_datetimes(
-      form_schema = fs,
-      verbose = verbose
-    ) %>%
-    { # nolint
+    odata_submission_rectangle(form_schema = fs,
+                               verbose = verbose) %>%
+    handle_ru_datetimes(form_schema = fs,
+                        verbose = verbose) %>%
+    {
+      # nolint
       if (download == TRUE) {
         fs::dir_create(local_dir)
         handle_ru_attachments(
@@ -264,11 +266,9 @@ odata_submission_get <- function(table = "Submissions",
         .
       }
     } %>%
-    handle_ru_geopoints(
-      form_schema = fs,
-      wkt = wkt,
-      verbose = verbose
-    ) %>%
+    handle_ru_geopoints(form_schema = fs,
+                        wkt = wkt,
+                        verbose = verbose) %>%
     handle_ru_geotraces(
       form_schema = fs,
       wkt = wkt,

diff --git a/tests/fixtures/test_odata_submission_get0.yml b/tests/fixtures/test_odata_submission_get0.yml
@@ -1,7 +1,7 @@
 http_interactions:
 - request:
     method: get
-    uri: https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/Submissions?%24skip=&%24top=&%24count=false&%24wkt=false
+    uri: https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/Submissions?%24count=false&%24wkt=false
     body:
       encoding: ''
       string: ''
@@ -17,7 +17,7 @@ http_interactions:
     headers:
       content-encoding: gzip
       content-type: application/json; charset=utf-8
-      date: Thu, 13 May 2021 05:10:48 GMT
+      date: Fri, 04 Jun 2021 02:29:36 GMT
       odata-version: '4.0'
       server: nginx
       strict-transport-security:
@@ -31,7 +31,7 @@ http_interactions:
       file: no
       string: '{"value":[{"__id":"uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28","encounter_start_datetime":"2020-11-02T12:06:19.314+08:00","reporter":null,"device_id":"5afb51f35ba0c572","encounter_end_datetime":"2020-11-02T12:15:48.463+08:00","__system":{"submissionDate":"2020-11-02T04:19:45.451Z","submitterId":"53","submitterName":"ruODK","attachmentsPresent":5,"attachmentsExpected":5,"status":null},"meta":{"instanceID":"uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28"},"location":{"area_name":"Test
         site 1","quadrat_photo":"1604290006239.jpg","corner1":{"type":"Point","coordinates":[115.8843989,-31.9959554,-12.699999809265137],"properties":{"accuracy":4.625}}},"habitat":{"morphological_type":"flat","morphological_type_photo":"1604290049411.jpg"},"vegetation_stratum@odata.navigationLink":"Submissions(''uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28'')/vegetation_stratum","perimeter":{"corner2":{"type":"Point","coordinates":[115.8842726,-31.9957893,-21.19999885559082],"properties":{"accuracy":4.861}},"corner3":{"type":"Point","coordinates":[115.8843031,-31.9958188,-21.19999885559082],"properties":{"accuracy":4.655}},"corner4":{"type":"Point","coordinates":[115.8845949,-31.9961397,-12.699999809265137],"properties":{"accuracy":4.913}},"mudmap_photo":"1604290379613.jpg"},"taxon_encounter@odata.navigationLink":"Submissions(''uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28'')/taxon_encounter"}],"@odata.context":"https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/$metadata#Submissions"}'
-  recorded_at: 2021-05-13 05:10:49 GMT
+  recorded_at: 2021-06-04 02:29:37 GMT
   recorded_with: vcr/0.6.0, webmockr/0.8.0
 - request:
     method: get
@@ -51,7 +51,7 @@ http_interactions:
     headers:
       content-encoding: gzip
       content-type: application/json; charset=utf-8
-      date: Thu, 13 May 2021 05:10:48 GMT
+      date: Fri, 04 Jun 2021 02:29:36 GMT
       etag: W/"bdf-C11XpUCPNjdiK9lUpOOOsqklGbw"
       server: nginx
       strict-transport-security:
@@ -64,5 +64,5 @@ http_interactions:
       encoding: UTF-8
       file: no
       string: '[{"path":"/meta","name":"meta","type":"structure","binary":null},{"path":"/meta/instanceID","name":"instanceID","type":"string","binary":null},{"path":"/encounter_start_datetime","name":"encounter_start_datetime","type":"dateTime","binary":null},{"path":"/reporter","name":"reporter","type":"string","binary":null},{"path":"/device_id","name":"device_id","type":"string","binary":null},{"path":"/location","name":"location","type":"structure","binary":null},{"path":"/location/area_name","name":"area_name","type":"string","binary":null},{"path":"/location/quadrat_photo","name":"quadrat_photo","type":"binary","binary":true},{"path":"/location/corner1","name":"corner1","type":"geopoint","binary":null},{"path":"/habitat","name":"habitat","type":"structure","binary":null},{"path":"/habitat/morphological_type","name":"morphological_type","type":"select1","binary":null},{"path":"/habitat/morphological_type_photo","name":"morphological_type_photo","type":"binary","binary":true},{"path":"/vegetation_stratum","name":"vegetation_stratum","type":"repeat","binary":null},{"path":"/vegetation_stratum/nvis_level3_broad_floristic_group","name":"nvis_level3_broad_floristic_group","type":"select1","binary":null},{"path":"/vegetation_stratum/max_height_m","name":"max_height_m","type":"decimal","binary":null},{"path":"/vegetation_stratum/foliage_cover","name":"foliage_cover","type":"select1","binary":null},{"path":"/vegetation_stratum/dominant_species_1","name":"dominant_species_1","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_2","name":"dominant_species_2","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_3","name":"dominant_species_3","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_4","name":"dominant_species_4","type":"string","binary":null},{"path":"/perimeter","name":"perimeter","type":"structure","binary":null},{"path":"/perimeter/corner2","name":"corner2","type":"geopoint","binary":null},{"path":"/perimeter/corner3","name":"corner3","type":"geopoint","binary":null},{"path":"/perimeter/corner4","name":"corner4","type":"geopoint","binary":null},{"path":"/perimeter/mudmap_photo","name":"mudmap_photo","type":"binary","binary":true},{"path":"/taxon_encounter","name":"taxon_encounter","type":"repeat","binary":null},{"path":"/taxon_encounter/field_name","name":"field_name","type":"string","binary":null},{"path":"/taxon_encounter/photo_in_situ","name":"photo_in_situ","type":"binary","binary":true},{"path":"/taxon_encounter/taxon_encounter_location","name":"taxon_encounter_location","type":"geopoint","binary":null},{"path":"/taxon_encounter/life_form","name":"life_form","type":"select1","binary":null},{"path":"/taxon_encounter/voucher_specimen_barcode","name":"voucher_specimen_barcode","type":"barcode","binary":null},{"path":"/taxon_encounter/voucher_specimen_label","name":"voucher_specimen_label","type":"string","binary":null},{"path":"/encounter_end_datetime","name":"encounter_end_datetime","type":"dateTime","binary":null}]'
-  recorded_at: 2021-05-13 05:10:49 GMT
+  recorded_at: 2021-06-04 02:29:37 GMT
   recorded_with: vcr/0.6.0, webmockr/0.8.0
diff --git a/tests/fixtures/test_odata_submission_get1.yml b/tests/fixtures/test_odata_submission_get1.yml
@@ -17,7 +17,7 @@ http_interactions:
     headers:
       content-encoding: gzip
       content-type: application/json; charset=utf-8; odata.metadata=minimal
-      date: Thu, 13 May 2021 05:10:52 GMT
+      date: Fri, 04 Jun 2021 02:29:43 GMT
       etag: W/"16d-WLGPb41H8z09a+jpiYUtvCEFcpA"
       odata-version: '4.0'
       server: nginx
@@ -30,5 +30,5 @@ http_interactions:
       encoding: UTF-8
       file: no
       string: '{"@odata.context":"https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/$metadata","value":[{"name":"Submissions","kind":"EntitySet","url":"Submissions"},{"name":"Submissions.vegetation_stratum","kind":"EntitySet","url":"Submissions.vegetation_stratum"},{"name":"Submissions.taxon_encounter","kind":"EntitySet","url":"Submissions.taxon_encounter"}]}'
-  recorded_at: 2021-05-13 05:10:52 GMT
+  recorded_at: 2021-06-04 02:29:43 GMT
   recorded_with: vcr/0.6.0, webmockr/0.8.0
diff --git a/tests/fixtures/test_odata_submission_get10.yml b/tests/fixtures/test_odata_submission_get10.yml
@@ -1,7 +1,7 @@
 http_interactions:
 - request:
     method: get
-    uri: https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/Submissions?%24skip=&%24top=&%24count=false&%24wkt=false&%24filter=year%28__system%2FsubmissionDate%29%20eq%202020
+    uri: https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/Submissions?%24count=false&%24wkt=false&%24filter=year%28__system%2FsubmissionDate%29%20eq%202020
     body:
       encoding: ''
       string: ''
@@ -17,7 +17,7 @@ http_interactions:
     headers:
       content-encoding: gzip
       content-type: application/json; charset=utf-8
-      date: Fri, 14 May 2021 03:59:49 GMT
+      date: Fri, 04 Jun 2021 02:29:54 GMT
       odata-version: '4.0'
       server: nginx
       strict-transport-security:
@@ -31,7 +31,7 @@ http_interactions:
       file: no
       string: '{"value":[{"__id":"uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28","encounter_start_datetime":"2020-11-02T12:06:19.314+08:00","reporter":null,"device_id":"5afb51f35ba0c572","encounter_end_datetime":"2020-11-02T12:15:48.463+08:00","__system":{"submissionDate":"2020-11-02T04:19:45.451Z","submitterId":"53","submitterName":"ruODK","attachmentsPresent":5,"attachmentsExpected":5,"status":null},"meta":{"instanceID":"uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28"},"location":{"area_name":"Test
         site 1","quadrat_photo":"1604290006239.jpg","corner1":{"type":"Point","coordinates":[115.8843989,-31.9959554,-12.699999809265137],"properties":{"accuracy":4.625}}},"habitat":{"morphological_type":"flat","morphological_type_photo":"1604290049411.jpg"},"vegetation_stratum@odata.navigationLink":"Submissions(''uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28'')/vegetation_stratum","perimeter":{"corner2":{"type":"Point","coordinates":[115.8842726,-31.9957893,-21.19999885559082],"properties":{"accuracy":4.861}},"corner3":{"type":"Point","coordinates":[115.8843031,-31.9958188,-21.19999885559082],"properties":{"accuracy":4.655}},"corner4":{"type":"Point","coordinates":[115.8845949,-31.9961397,-12.699999809265137],"properties":{"accuracy":4.913}},"mudmap_photo":"1604290379613.jpg"},"taxon_encounter@odata.navigationLink":"Submissions(''uuid:469f71d3-d7aa-4c74-8aaa-af5f667a2f28'')/taxon_encounter"}],"@odata.context":"https://odkc.dbca.wa.gov.au/v1/projects/2/forms/Flora-Quadrat-04.svc/$metadata#Submissions"}'
-  recorded_at: 2021-05-14 03:59:50 GMT
+  recorded_at: 2021-06-04 02:29:56 GMT
   recorded_with: vcr/0.6.0, webmockr/0.8.0
 - request:
     method: get
@@ -51,7 +51,7 @@ http_interactions:
     headers:
       content-encoding: gzip
       content-type: application/json; charset=utf-8
-      date: Fri, 14 May 2021 03:59:49 GMT
+      date: Fri, 04 Jun 2021 02:29:55 GMT
       etag: W/"bdf-C11XpUCPNjdiK9lUpOOOsqklGbw"
       server: nginx
       strict-transport-security:
@@ -64,5 +64,5 @@ http_interactions:
       encoding: UTF-8
       file: no
       string: '[{"path":"/meta","name":"meta","type":"structure","binary":null},{"path":"/meta/instanceID","name":"instanceID","type":"string","binary":null},{"path":"/encounter_start_datetime","name":"encounter_start_datetime","type":"dateTime","binary":null},{"path":"/reporter","name":"reporter","type":"string","binary":null},{"path":"/device_id","name":"device_id","type":"string","binary":null},{"path":"/location","name":"location","type":"structure","binary":null},{"path":"/location/area_name","name":"area_name","type":"string","binary":null},{"path":"/location/quadrat_photo","name":"quadrat_photo","type":"binary","binary":true},{"path":"/location/corner1","name":"corner1","type":"geopoint","binary":null},{"path":"/habitat","name":"habitat","type":"structure","binary":null},{"path":"/habitat/morphological_type","name":"morphological_type","type":"select1","binary":null},{"path":"/habitat/morphological_type_photo","name":"morphological_type_photo","type":"binary","binary":true},{"path":"/vegetation_stratum","name":"vegetation_stratum","type":"repeat","binary":null},{"path":"/vegetation_stratum/nvis_level3_broad_floristic_group","name":"nvis_level3_broad_floristic_group","type":"select1","binary":null},{"path":"/vegetation_stratum/max_height_m","name":"max_height_m","type":"decimal","binary":null},{"path":"/vegetation_stratum/foliage_cover","name":"foliage_cover","type":"select1","binary":null},{"path":"/vegetation_stratum/dominant_species_1","name":"dominant_species_1","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_2","name":"dominant_species_2","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_3","name":"dominant_species_3","type":"string","binary":null},{"path":"/vegetation_stratum/dominant_species_4","name":"dominant_species_4","type":"string","binary":null},{"path":"/perimeter","name":"perimeter","type":"structure","binary":null},{"path":"/perimeter/corner2","name":"corner2","type":"geopoint","binary":null},{"path":"/perimeter/corner3","name":"corner3","type":"geopoint","binary":null},{"path":"/perimeter/corner4","name":"corner4","type":"geopoint","binary":null},{"path":"/perimeter/mudmap_photo","name":"mudmap_photo","type":"binary","binary":true},{"path":"/taxon_encounter","name":"taxon_encounter","type":"repeat","binary":null},{"path":"/taxon_encounter/field_name","name":"field_name","type":"string","binary":null},{"path":"/taxon_encounter/photo_in_situ","name":"photo_in_situ","type":"binary","binary":true},{"path":"/taxon_encounter/taxon_encounter_location","name":"taxon_encounter_location","type":"geopoint","binary":null},{"path":"/taxon_encounter/life_form","name":"life_form","type":"select1","binary":null},{"path":"/taxon_encounter/voucher_specimen_barcode","name":"voucher_specimen_barcode","type":"barcode","binary":null},{"path":"/taxon_encounter/voucher_specimen_label","name":"voucher_specimen_label","type":"string","binary":null},{"path":"/encounter_end_datetime","name":"encounter_end_datetime","type":"dateTime","binary":null}]'
-  recorded_at: 2021-05-14 03:59:50 GMT
+  recorded_at: 2021-06-04 02:29:56 GMT
   recorded_with: vcr/0.6.0, webmockr/0.8.0