-
Notifications
You must be signed in to change notification settings - Fork 12
/
odata_submission_get.R
292 lines (280 loc) · 11.3 KB
/
odata_submission_get.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
#' Retrieve and rectangle form submissions, parse dates, geopoints, download and
#' link attachments.
#'
#' `r lifecycle::badge("stable")`
#'
#' @details \code{\link{odata_submission_get}} downloads submissions from
#' (default) the main form group (submission table) including any non-repeating
#' form groups, or from any other table as specified by parameter `table`.
#'
#' With parameter \code{parse=TRUE} (default), submission data is parsed into a
#' tibble. Any fields of type \code{dateTime} or \code{date} are parsed into
#' dates, with an optional parameter \code{tz} to specify the local timezone.
#'
#' A parameter \code{local_dir} (default: \code{media}) specifies a local
#' directory for downloaded attachment files.
#' Already existing, previously downloaded attachments will be retained.
#'
#' With parameter `wkt=TRUE`, spatial fields will be returned as WKT, rather
#' than GeoJSON. In addition, fields of type `geopoint` will be split into
#' latitude, longitude, and altitude, prefixed with the original field name.
#' E.g. a field `start_location` of type `geopoint` will be split into
#' `start_location_latitude`, `start_location_longitude`, and
#' `start_location_altitude`. The field name prefix will allow multiple fields
#' of type `geopoint` to be split into their components without naming
#' conflicts.
#'
#' Geotraces (lines) and gepshapes (polygons) will be retained in their original
#' format, plus columns of their first point's coordinate components as
#' provided by \code{\link{split_geotrace}} and \code{\link{split_geoshape}},
#' respectively.
#'
#' Entirely unpopulated form fields, as well as notes and form groups, will be
#' excluded from the resulting tibble.
#' Submitting at least one complete form instance will prevent the accidental
#' exclusion of an otherwise mostly empty form field.
#'
#' The only remaining manual step is to optionally join any sub-tables to the
#' master table.
#'
#' The parameter \code{verbose} enables diagnostic messages along the download
#' and parsing process.
#'
#' With parameter \code{parse=FALSE}, submission data is presented as nested
#' list, which is the R equivalent of the JSON structure returned from the API.
#' From there, \code{\link{odata_submission_rectangle}} can rectangle the data
#' into a tibble, and subsequent lines of \code{\link{handle_ru_datetimes}},
#' \code{\link{handle_ru_attachments}}, \code{\link{handle_ru_geopoints}},
#' \code{\link{handle_ru_geotraces}}, and \code{\link{handle_ru_geoshapes}}
#' parse dates, download and link file attachments, and extract coordinates from
#' geofields.
#' \code{ruODK} offers this manual and explicit pathway as an option to
#' investigate and narrow down unexpected or unwanted behaviour.
#'
#' @param table The submission EntityType, or in plain words, the table name.
#' Default: \code{Submissions} (the main table).
#' Change to \code{Submissions.GROUP_NAME} for repeating form groups.
#' The group name can be found through \code{\link{odata_service_get}}.
#' @param skip The number of rows to be omitted from the results.
#' Example: 10, default: \code{NA} (none skipped).
#' @param top The number of rows to return.
#' Example: 100, default: \code{NA} (all returned).
#' @param count If TRUE, an \code{@odata.count} property will be returned in the
#' response from ODK Central. Default: \code{FALSE}.
#' @param wkt If TRUE, geospatial data will be returned as WKT (Well Known Text)
#' strings. Default: \code{FALSE}, returns GeoJSON structures.
#' Note that accuracy is only returned through GeoJSON.
#' @param filter If provided, will filter responses to those matching the query.
#' For an `odkc_version` below 1.1, this parameter will be discarded.
#' In ODK Central v1.1, only the fields `system/submitterId` and
#' `system/submissionDate` are available to reference.
#' In ODK Central v1.2, other fields may become available.
#' The operators `lt`, `lte`, `eq`, `neq`, `gte`, `gt`, `not`, `and`, and `or`
#' are supported, and the built-in functions
#' `now`, `year`, `month`, `day`, `hour`, `minute`, `second.`
#' `ruODK` does not validate the query string given to `filter`.
#' It is highly recommended to refer to the ODK Central API documentation
#' as well as the
#' [OData spec on filters](https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358948).
#' for filter options and capabilities
#' @param parse Whether to parse submission data based on form schema.
#' Dates and datetimes will be parsed into local time.
#' Attachments will be downloaded, and the field updated to the local file
#' path.
#' Point locations will be split into components; GeoJSON (\code{wkt=FALSE})
#' will be split into latitude, longitude, altitude and accuracy
#' (with anonymous field names), while WKT will be split into
#' longitude, latitude,and altitude (missing accuracy) prefixed by
#' the original field name.
#' See details for the handling of geotraces and geoshapes.
#' Default: TRUE.
#' @param download Whether to download attachments to \code{local_dir} or not.
#' If in the future ODK Central supports hot-linking attachments,
#' this parameter will replace attachment file names with their fully
#' qualified attachment URL.
#' Default: TRUE.
#' @param orders (vector of character) Orders of datetime elements for
#' lubridate.
#' Default:
#' \code{c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd")}.
#' @param local_dir The local folder to save the downloaded files to,
#' default: \code{"media"}.
#' @template param-pid
#' @template param-fid
#' @template param-url
#' @template param-auth
#' @template param-odkcv
#' @template param-tz
#' @template param-retries
#' @template param-verbose
#' @return A list of lists.
#' \itemize{
#' \item \code{value} contains the submissions as list of lists.
#' \item \code{@odata.context} is the URL of the metadata.
#' \item \code{@odata.count} is the total number of rows in the table.
#' }
# nolint start
#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/odata-endpoints/odata-form-service}
#' @seealso \url{https://odkcentral.docs.apiary.io/#reference/odata-endpoints/odata-form-service/data-document}
# nolint end
#' @family odata-api
#' @importFrom rlang %||%
#' @export
#' @examples
#' \dontrun{
#' # See vignette("setup") for setup and authentication options
#' # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...")
#'
#' form_tables <- ruODK::odata_service_get()
#' data <- odata_submission_get() # default: main data table
#' data <- odata_submission_get(table = form_tables$url[1]) # same, explicitly
#' data_sub1 <- odata_submission_get(table = form_tables$url[2]) # sub-table 1
#' data_sub2 <- odata_submission_get(table = form_tables$url[3]) # sub-table 2
#'
#' # Skip one row, return the next 1 rows (top), include total row count
#' data <- odata_submission_get(
#' table = form_tables$url[1],
#' skip = 1,
#' top = 1,
#' count = TRUE
#' )
#'
#' # Filter submissions
#' data <- odata_submission_get(
#' table = form_tables$url[1],
#' filter = "year(__system/submissionDate) lt year(now())"
#' )
#' data <- odata_submission_get(
#' table = form_tables$url[1],
#' filter = "year(__system/submissionDate) lt 2020"
#' )
#' }
odata_submission_get <- function(table = "Submissions",
skip = NULL,
top = NULL,
count = FALSE,
wkt = FALSE,
filter = NULL,
parse = TRUE,
download = TRUE,
orders = c(
"YmdHMS",
"YmdHMSz",
"Ymd HMS",
"Ymd HMSz",
"Ymd",
"ymd"
),
local_dir = "media",
pid = get_default_pid(),
fid = get_default_fid(),
url = get_default_url(),
un = get_default_un(),
pw = get_default_pw(),
odkc_version = get_default_odkc_version(),
tz = get_default_tz(),
retries = get_retries(),
verbose = get_ru_verbose()) {
yell_if_missing(url, un, pw)
#----------------------------------------------------------------------------#
# Download submissions
ru_msg_info("Downloading submissions...", verbose = verbose)
qry <- list(
`$skip` = skip %||% "",
`$top` = top %||% "",
`$count` = ifelse(count == FALSE, "false", "true"),
`$wkt` = ifelse(wkt == FALSE, "false", "true")
# `$filter` = ifelse(odkc_version>=1.1, filter %||% "", "")
)
if(odkc_version>=1.1 && !is.null(filter) && filter != "") {
qry$`$filter` <- as.character(filter)
}
sub <- httr::RETRY(
"GET",
httr::modify_url(
url,
path = glue::glue(
"v1/projects/{pid}/forms/{URLencode(fid, reserved = TRUE)}.svc/{table}"
)
),
query = qry,
times = retries,
httr::add_headers(Accept = "application/json"),
httr::authenticate(un, pw)
) %>%
yell_if_error(., url, un, pw) %>%
httr::content(.)
ru_msg_success("Downloaded submissions.", verbose = verbose)
if (parse == FALSE) {
ru_msg_success("Returning unparsed submissions.", verbose = verbose)
return(sub)
}
#----------------------------------------------------------------------------#
# Get form schema
ru_msg_info("Reading form schema...", verbose = verbose)
fs <- form_schema(
parse = TRUE,
pid = pid,
fid = fid,
url = url,
un = un,
pw = pw,
odkc_version = odkc_version,
retries = retries
)
#----------------------------------------------------------------------------#
# Parse submission data
ru_msg_info("Parsing submissions...", verbose = verbose)
# Rectangle, handle date/times, attachments, geopoints, geotraces, geoshapes
sub <- sub %>%
odata_submission_rectangle(
form_schema = fs,
verbose = verbose
) %>%
handle_ru_datetimes(
form_schema = fs,
verbose = verbose
) %>%
{ # nolint
if (download == TRUE) {
fs::dir_create(local_dir)
handle_ru_attachments(
data = .,
form_schema = fs,
local_dir = local_dir,
pid = pid,
fid = fid,
url = url,
un = un,
pw = pw,
retries = retries,
verbose = verbose
)
} else {
.
}
} %>%
handle_ru_geopoints(
form_schema = fs,
wkt = wkt,
verbose = verbose
) %>%
handle_ru_geotraces(
form_schema = fs,
wkt = wkt,
verbose = verbose,
odkc_version = odkc_version
) %>%
handle_ru_geoshapes(
form_schema = fs,
wkt = wkt,
verbose = verbose,
odkc_version = odkc_version
)
#
# End parse submission data
#----------------------------------------------------------------------------#
ru_msg_success("Returning parsed submissions.", verbose = verbose)
sub
}
# usethis::use_test("odata_submission_get") # nolint