update tests

.Rbuildignore

@@ -5,3 +5,4 @@
 ^README\.en\.md$
 ^cran-comments\.md$
 ^appveyor\.yml$
+^\.appId$

.gitignore

@@ -1,3 +1,4 @@
 .Rproj.user
 .Rhistory
 .RData
+.appId

R/getStatsData.R

@@ -62,32 +62,19 @@ estat_getStatsData <- function(appId, statsDataId,
                                ...)
 {
   result <- list()
-  LIMIT <- 100000
 
-  if (.aquire_all) {
-    record_count <- estat_getStatsDataCount(appId, statsDataId, ...)
-    if (is.null(limit)) {
-      max_count <- limit <- record_count
-    } else {
-      max_count <- min(limit, record_count)
-    }
-
-    starts <- seq(from = 1, to = max_count, by = LIMIT)
-    limits <- c(rep(LIMIT, length(starts) - 1), limit %% LIMIT)
-  } else {
-    starts <- startPosition
-    limits <- limit
-  }
+  record_count <- estat_getStatsDataCount(appId, statsDataId, ...)
+  ranges <- calc_ranges(startPosition, limit, record_count, .aquire_all)
 
-  for (i in seq_along(starts)) {
+  for (i in seq_along(ranges$starts)) {
     message(sprintf("Aquiring %.0f records from %.0f (Total %.0f records)...\n",
-                    limits[i], starts[i], max_count))
+                    ranges$limits[i], ranges$starts[i], record_count))
 
     result_text <- estat_api("rest/2.1/app/getSimpleStatsData",
                              appId = appId,
                              statsDataId = statsDataId,
-                             startPosition = format(starts[i], scientific = FALSE),
-                             limit = format(limits[i], scientific = FALSE),
+                             startPosition = format(ranges$starts[i], scientific = FALSE),
+                             limit = format(ranges$limits[i], scientific = FALSE),
                              sectionHeaderFlg = 2, # Skip metadata section
                              ...)
 
@@ -105,7 +92,37 @@ estat_getStatsData <- function(appId, statsDataId,
   dplyr::bind_rows(result)
 }
 
+
 #' @rdname estat_getStatsData
 #' @export
 estat_getSimpleStatsData <- estat_getStatsData
 
+
+calc_ranges <- function(startPosition, limit, record_count, .aquire_all, .max_records_at_once = 100000) {
+  ranges <- list()
+
+  if (is.null(startPosition)) {
+    startPosition <- 1
+  }
+
+  if (is.null(limit)) {
+    endPosition <- record_count
+  } else {
+    endPosition <- min(startPosition + limit - 1, record_count)
+  }
+
+  if (.aquire_all) {
+    ranges$starts <- seq(from = startPosition, to = endPosition, by = .max_records_at_once)
+    ranges$limits <- rep(.max_records_at_once, length(ranges$starts))
+    # treat a fraction
+    fraction <- (endPosition - startPosition + 1) %% .max_records_at_once
+    if (fraction != 0) {
+      ranges$limits[length(ranges$limits)] <- fraction
+    }
+  } else {
+    ranges$starts <- startPosition
+    ranges$limits <- min(startPosition + .max_records_at_once - 1, endPosition)
+  }
+
+  ranges
+}

tests/testthat/test-api-real.R

@@ -0,0 +1,21 @@
+context("API with real request")
+
+if (!file.exists("../../.appId")) {
+  return()
+}
+
+appId <- readr::read_lines("../../.appId")
+
+test_that("estat_getStatsData without limit and startPosition, with >100000 records works fine", {
+  d <- estat_getStatsData(
+    appId = appId,
+    statsDataId = "0003103532",
+    cdCat01 = c("010800130","010800140")
+  )
+
+  expect_equal(nrow(d), 13184)
+  expect_equal(names(d), c("tab_code", "\u8868\u7ae0\u9805\u76ee", "cat01_code", "\u54c1\u76ee\u5206\u985e\uff0827\u5e74\u6539\u5b9a\uff09",
+                           "cat02_code", "\u4e16\u5e2f\u533a\u5206", "area_code", "\u5730\u57df\u533a\u5206",
+                           "time_code", "\u6642\u9593\u8ef8\uff08\u6708\u6b21\uff09",
+                           "unit", "value"))
+})

tests/testthat/test-api.R

@@ -1,12 +1,28 @@
-context("API")
+context("API with mocks")
 
-dummy_res <- structure(list(
-  content = raw(0),
-  url = '',
-  headers = list(`Content-Type` = "application/json;charset=utf-8"),
-  status_code = 200
-),
-class = "response")
+dummy_res <- structure(
+  list(
+    url = "",
+    status_code = 200L,
+    headers = structure(
+      list(
+        `content-type` = "application/json;charset=utf-8",
+        `content-encoding` = "gzip",
+        `content-length` = "868"
+      ),
+      .Names = c("content-type",
+                 "content-encoding",
+                 "content-length"),
+      class = c("insensitive", "list")
+    ),
+    content = raw(0)
+  ),
+  .Names = c("url",
+             "status_code",
+             "headers",
+             "content"),
+  class = "response"
+)
 
 test_that("estat_getStatsList processes the API response as expected", {
   with_mock(

tests/testthat/test-calc-range.R

@@ -0,0 +1,32 @@
+context("calc_range")
+
+params <- list(
+  list(startPosition = NULL, limit = NULL, record_count = 100L, .aquire_all = FALSE, max = 1e+05,
+       expect = list(starts = 1, limits = 100)),
+  list(startPosition = NULL, limit = NULL, record_count = 100L, .aquire_all = TRUE, max = 1e+05,
+       expect = list(starts = 1, limits = 100)),
+  list(startPosition = NULL, limit = NULL, record_count = 100L, .aquire_all = FALSE, max = 10,
+       expect = list(starts = 1, limits = 10)),
+  list(startPosition = NULL, limit = NULL, record_count = 100L, .aquire_all = TRUE, max = 10,
+       expect = list(starts = c( 1, 11, 21, 31, 41, 51, 61, 71, 81, 91),
+                     limits = c(10, 10, 10, 10, 10, 10, 10, 10, 10, 10))),
+  list(startPosition = 11, limit = NULL, record_count = 100L, .aquire_all = TRUE, max = 10,
+       expect = list(starts = c(11, 21, 31, 41, 51, 61, 71, 81, 91),
+                     limits = c(10, 10, 10, 10, 10, 10, 10, 10, 10))),
+  list(startPosition = 2, limit = NULL, record_count = 100L, .aquire_all = TRUE, max = 10,
+       expect = list(starts = c( 2, 12, 22, 32, 42, 52, 62, 72, 82, 92),
+                     limits = c(10, 10, 10, 10, 10, 10, 10, 10, 10,  9)))
+)
+
+for (param in params) {
+  test_that("test calc_range", {
+    r <- estatapi:::calc_ranges(
+      startPosition = param$startPosition,
+      limit = param$limit,
+      record_count = param$record_count,
+      .aquire_all = param$.aquire_all,
+      .max_records_at_once = param$max
+    )
+    expect_equal(r, param$expect)
+  })
+}