Merge 7dcf574 into 1b630fc

Chicago · Feb 24, 2016 · 06d2c85 · 06d2c85
2 parents 1b630fc + 7dcf574
commit 06d2c85
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 47 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -10,8 +10,8 @@ Description: Provides easier interaction with
     format and manages throttling by 'Socrata'.
     Users can upload data to Socrata portals directly
     from R.
-Version: 1.7.1-8
-Date: 2016-02-13
+Version: 1.7.2-1
+Date: 2016-02-24
 Author: Hugh Devlin, Ph. D., Tom Schenk, Jr., and John Malc
 Maintainer: "Tom Schenk Jr." <developers@cityofchicago.org>
 Depends:
@@ -27,4 +27,4 @@ Suggests:
 License: MIT + file LICENSE
 URL: https://github.com/Chicago/RSocrata
 BugReports: https://github.com/Chicago/RSocrata/issues
-RoxygenNote: 5.0.1
+RoxygenNote: 5.0.1
diff --git a/R/RSocrata.R b/R/RSocrata.R
@@ -107,11 +107,22 @@ fieldName <- function(humanName) {
 posixify <- function(x) {
 	x <- as.character(x)
 	if (length(x)==0) return(x)
-	# Two calendar date formats supplied by Socrata
-	if(any(regexpr("^[[:digit:]]{1,2}/[[:digit:]]{1,2}/[[:digit:]]{4}$", x[1])[1] == 1))
-	  strptime(x, format="%m/%d/%Y") # short date format
-	else
-	  strptime(x, format="%m/%d/%Y %I:%M:%S %p") # long date-time format 
+
+	## Define regex patterns for short and long date formats, which are the two 
+	## formats that are supplied by Socrata. 
+	patternShort <- paste0("^[[:digit:]]{1,2}/[[:digit:]]{1,2}/[[:digit:]]{4}$")
+	patternLong <- paste0("^[[:digit:]]{1,2}/[[:digit:]]{1,2}/[[:digit:]]{4}",
+	                      "[[:digit:]]{1,2}:[[:digit:]]{1,2}:[[:digit:]]{1,2}",
+	                      "AM|PM", "$")
+	## Find number of matches with grep
+	nMatchesShort <- grep(pattern = patternShort, x)
+	nMatchesLong <- grep(pattern = patternLong, x)
+	## Parse as the most likely calendar date format. Ties go to short format.
+	if(length(nMatchesLong) > length(nMatchesShort)){
+	  as.POSIXct(strptime(x, format="%m/%d/%Y %I:%M:%S %p")) # long date-time format
+	}	else {
+	  as.POSIXct(strptime(x, format="%m/%d/%Y")) # short date format
+	}
 }
 
 #' Convert Socrata money fields to numeric

diff --git a/tests/testthat/test-all.R b/tests/testthat/test-all.R
@@ -11,26 +11,67 @@ socrataPassword <- Sys.getenv("SOCRATA_PASSWORD", "7vFDsGFDUG")
 context("posixify function")
 
 test_that("posixify returns Long format", {
+  df <- read.socrata('http://soda.demo.socrata.com/resource/4334-bgaj.csv')
   dt <- posixify("09/14/2012 10:38:01 PM")
-  expect_equal("POSIXlt", class(dt)[1], label="first data type of a date")
-  expect_equal(2012, dt$year + 1900, label="year")
-  expect_equal(9, dt$mon + 1, label="month")
-  expect_equal(14, dt$mday, label="day")
-  expect_equal(22, dt$hour, label="hours")
-  expect_equal(38, dt$min, label="minutes")
-  expect_equal(1, dt$sec, label="seconds")
+  expect_equal(dt, df$Datetime[1])  ## Check that download matches test
+
+  expect_equal("POSIXct", class(dt)[1], label="Long format date data type")
+  expect_equal("2012", format(dt, "%Y"), label="year")
+  expect_equal("09", format(dt, "%m"), label="month")
+  expect_equal("14", format(dt, "%d"), label="day")
+  expect_equal("22", format(dt, "%H"), label="hours")
+  expect_equal("38", format(dt, "%M"), label="minutes")
+  expect_equal("01", format(dt, "%S"), label="seconds")
 })
 
 
 test_that("posixify returns Short format", {
   dt <- posixify("09/14/2012")
-  expect_equal("POSIXlt", class(dt)[1], label="first data type of a date")
-  expect_equal(2012, dt$year + 1900, label="year")
-  expect_equal(9, dt$mon + 1, label="month")
-  expect_equal(14, dt$mday, label="day")
-  expect_equal(0, dt$hour, label="hours")
-  expect_equal(0, dt$min, label="minutes")
-  expect_equal(0, dt$sec, label="seconds")
+  expect_equal("POSIXct", class(dt)[1], label="Short format date data type")
+  expect_equal("2012", format(dt, "%Y"), label="year")
+  expect_equal("09", format(dt, "%m"), label="month")
+  expect_equal("14", format(dt, "%d"), label="day")
+  expect_equal("00", format(dt, "%H"), label="hours")
+  expect_equal("00", format(dt, "%M"), label="minutes")
+  expect_equal("00", format(dt, "%S"), label="seconds")
+})
+
+context("Socrata Calendar")
+
+test_that("Calendar Date Short", {
+  df <- read.socrata('http://data.cityofchicago.org/resource/y93d-d9e3.csv?$order=debarment_date')
+  dt <- df$DEBARMENT.DATE[1] # "05/21/1981"
+  expect_equal("POSIXct", class(dt)[1], label="data type of a date")
+  expect_equal("81", format(dt, "%y"), label="year")
+  expect_equal("05", format(dt, "%m"), label="month")
+  expect_equal("21", format(dt, "%d"), label="day")
+  expect_equal("00", format(dt, "%H"), label="hours")
+  expect_equal("00", format(dt, "%M"), label="minutes")
+  expect_equal("00", format(dt, "%S"), label="seconds")
+})
+
+test_that("Date is not entirely NA if the first record is bad (issue 68)", {
+
+  ## Define and test issue 68
+  # df <- read.socrata('http://data.cityofchicago.org/resource/me59-5fac.csv')
+  # expect_false(object = all(is.na(df$Creation.Date)),
+  #              "Testing issue 68 https://github.com/Chicago/RSocrata/issues/68")
+
+  df <- read.socrata("https://data.cityofchicago.org/resource/4h87-zdcp.csv")
+  expect_false(object = all(is.na(df$DATE.RECEIVED)),
+               "Testing issue 68 https://github.com/Chicago/RSocrata/issues/68")
+
+
+  ## Define smaller tests
+  dates_clean <- posixify(c("01/01/2011", "01/01/2011", "01/01/2011"))
+  dates_mixed <- posixify(c("Date", "01/01/2011", "01/01/2011"))
+  dates_dirty <- posixify(c("Date", "junk", "junk"))
+
+  ## Execute smaller tests
+  expect_true(all(!is.na(dates_clean)))  ## Nothing should be NA
+  expect_true(any(is.na(dates_mixed)))   ## Some should be NA
+  expect_true(any(!is.na(dates_mixed)))  ## Some should not be NA
+  expect_true(all(is.na(dates_dirty)))   ## Everything should be NA
 })
 
 context("change money to numeric")
@@ -94,31 +135,6 @@ test_that("format is not supported", {
   expect_error(read.socrata('http://soda.demo.socrata.com/resource/4334-bgaj.xml'))
 })
 
-context("Socrata Calendar")
-
-test_that("Calendar Date Long", {
-  df <- read.socrata('http://soda.demo.socrata.com/resource/4334-bgaj.csv')
-  dt <- df$Datetime[1] # "2012-09-14 22:38:01"
-  expect_equal("POSIXlt", class(dt)[1], label="data type of a date")
-  expect_equal(2012, dt$year + 1900, label="year")
-  expect_equal(9, dt$mon + 1, label="month")
-  expect_equal(14, dt$mday, label="day")
-  expect_equal(22, dt$hour, label="hours")
-  expect_equal(38, dt$min, label="minutes")
-  expect_equal(1, dt$sec, label="seconds")
-})
-
-test_that("Calendar Date Short", {
-  df <- read.socrata('http://data.cityofchicago.org/resource/y93d-d9e3.csv?$order=debarment_date')
-  dt <- df$DEBARMENT.DATE[1] # "05/21/1981"
-  expect_equal("POSIXlt", class(dt)[1], label="data type of a date")
-  expect_equal(81, dt$year, label="year")
-  expect_equal(5, dt$mon + 1, label="month")
-  expect_equal(21, dt$mday, label="day")
-  expect_equal(0, dt$hour, label="hours")
-  expect_equal(0, dt$min, label="minutes")
-  expect_equal(0, dt$sec, label="seconds")
-})
 
 context("Checks the validity of 4x4")