Merge 018fcaa into ac51fd7

frictionlessdata · Mar 30, 2020 · 48225e9 · 48225e9
2 parents ac51fd7 + 018fcaa
commit 48225e9
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 16 deletions.
diff --git a/data/data_infer_dates.csv b/data/data_infer_dates.csv
@@ -0,0 +1,20 @@
+first_name,last_name,email,gender,ip_address,date
+Roobbie,Wilbore,rwilbore0@bandcamp.com,Female,82.243.17.211,01/07/2018
+Aprilette,Dole,adole1@altervista.org,Female,12.132.123.75,03/27/2017
+Phillipe,Winkworth,pwinkworth2@wufoo.com,Male,71.62.119.246,01/20/2018
+Arvy,Lempke,alempke3@squidoo.com,Male,117.227.52.135,10/20/2017
+Cody,Jakov,cjakov4@i2i.jp,Female,172.197.66.100,05/23/2017
+Colan,Keggins,ckeggins5@timesonline.co.uk,Male,181.229.74.134,05/27/2017
+Nancee,Hembry,nhembry6@reddit.com,Female,156.241.166.133,02/16/2017
+Marla,Kopta,mkopta7@youku.com,Female,7.62.98.38,01/11/2018
+Sharlene,Roll,sroll8@hexun.com,Female,33.240.2.44,12/30/2017
+Misti,Fillan,mfillan9@friendfeed.com,Female,116.231.5.10,02/21/2017
+Elyse,McSkin,emcskina@naver.com,Female,174.104.235.87,12/27/2017
+Avril,Harm,aharmb@bloomberg.com,Female,69.223.89.103,07/15/2017
+Viola,Lopez,vlopezc@exblog.jp,Female,166.108.241.67,03/1/2017
+Dulci,Lutz,dlutzd@newyorker.com,Female,68.222.192.251,03/30/2017
+Mallory,Ivett,mivette@slideshare.net,Male,54.169.32.87,06/05/2017
+Gris,Attree,gattreef@cpanel.net,Male,33.79.140.56,07/26/2017
+Jackelyn,Matas,jmatasg@cocolog-nifty.com,Female,91.95.85.200,11/16/2017
+Meggi,Corey,mcoreyh@virginia.edu,Female,23.132.198.151,01/13/2018
+Chuck,Ditchfield,cditchfieldi@networksolutions.com,Male,109.14.57.171,03/31/2017
diff --git a/package.json b/package.json
@@ -34,7 +34,6 @@
     "axios": "^0.19.0",
     "csv-parse": "^4.8.8",
     "csv-sniffer": "^0.1.1",
-    "d3-time-format": "^0.3.2",
     "es6-error": "^4.0.2",
     "lodash": "^4.17.15",
     "moment": "~2.21.0",

diff --git a/src/helpers.js b/src/helpers.js
@@ -69,6 +69,52 @@ function expandFieldDescriptor(descriptor) {
   return descriptor
 }
 
+// Date/time conversion
+// https://gist.github.com/jamesguan/5d54454c1a88f63c30cf3c5fb4d0fc20
+
+/*
+ * Description: Convert a python format string to javascript format string
+ * Example:     "%m/%d/%Y" to "MM/DD/YYYY"
+ * @param:  formatStr is the python format string
+ * @return: the javascript format string
+ */
+function convertDatetimeFormatFromFDtoJS(formatStr) {
+  for (const key in DATETIME_FORMATS_MAP_FROM_FD_TO_JS) {
+    formatStr = formatStr.split(key).join(DATETIME_FORMATS_MAP_FROM_FD_TO_JS[key])
+  }
+  return formatStr
+}
+
+/* Key: Python format
+ * Value: Javascript format
+ */
+const DATETIME_FORMATS_MAP_FROM_FD_TO_JS = Object.freeze({
+  '%A': 'dddd', // Weekday as locale’s full name: (In English: Sunday, .., Saturday)(Auf Deutsch: Sonntag, .., Samstag)
+  '%a': 'ddd', // Weekday abbreivated: (In English: Sun, .., Sat)(Auf Deutsch: So, .., Sa)
+  '%B': 'MMMM', // Month name: (In English: January, .., December)(Auf Deutsch: Januar, .., Dezember)
+  '%b': 'MMM', // Month name abbreviated: (In English: Jan, .., Dec)(Auf Deutsch: Jan, .., Dez)
+  '%c': 'ddd MMM DD HH:mm:ss YYYY', // Locale’s appropriate date and time representation: (English: Sun Oct 13 23:30:00 1996)(Deutsch: So 13 Oct 22:30:00 1996)
+  '%d': 'DD', // Day 0 padded: (01, .., 31)
+  '%f': 'SSS', // Microseconds 0 padded: (000000, .., 999999)
+  '%H': 'HH', // Hour (24-Hour) 0 padded: (00, .., 23)
+  '%I': 'hh', // Hour (12-Hour) 0 padded: (01, .., 12)
+  '%j': 'DDDD', // Day of Year 0 padded: (001, .., 366)
+  '%M': 'mm', // Minute 0 padded: (01, .. 59)
+  '%m': 'MM', // Month 0 padded: (01, .., 12)
+  '%p': 'A', // Locale equivalent of AM/PM: (EN: AM, PM)(DE: am, pm)
+  '%S': 'ss', // Second 0 padded: (00, .., 59)
+  '%U': 'ww', // Week # of Year (Sunday): (00, .., 53)  All days in a new year preceding the first Sunday are considered to be in week 0.
+  '%W': 'ww', // Week # of Year (Monday): (00, .., 53)  All days in a new year preceding the first Monday are considered to be in week 0.
+  '%w': 'd', // Weekday as #: (0, 6)
+  '%X': 'HH:mm:ss', // Locale's appropriate time representation: (EN: 23:30:00)(DE: 23:30:00)
+  '%x': 'MM/DD/YYYY', // Locale's appropriate date representation: (None: 02/14/16)(EN: 02/14/16)(DE: 14.02.16)
+  '%Y': 'YYYY', // Year as #: (1970, 2000, 2038, 292,277,026,596)
+  '%y': 'YY', // Year without century 0 padded: (00, .., 99)
+  '%Z': 'z', // Time zone name: ((empty), UTC, EST, CST) (empty string if the object is naive).
+  '%z': 'ZZ', // UTC offset in the form +HHMM or -HHMM: ((empty), +0000, -0400, +1030) Empty string if the the object is naive.
+  '%%': '%', // A literal '%' character: (%)
+})
+
 // Miscellaneous
 
 function isRemotePath(path) {
@@ -82,5 +128,6 @@ module.exports = {
   retrieveDescriptor,
   expandSchemaDescriptor,
   expandFieldDescriptor,
+  convertDatetimeFormatFromFDtoJS,
   isRemotePath,
 }
diff --git a/src/schema.js b/src/schema.js
@@ -1,4 +1,5 @@
 const fs = require('fs')
+const moment = require('moment')
 const min = require('lodash/min')
 const zip = require('lodash/zip')
 const isArray = require('lodash/isArray')
@@ -7,7 +8,6 @@ const isString = require('lodash/isString')
 const cloneDeep = require('lodash/cloneDeep')
 const isBoolean = require('lodash/isBoolean')
 const upperFirst = require('lodash/upperFirst')
-const { timeParse } = require('d3-time-format')
 const { TableSchemaError } = require('./errors')
 const { Profile } = require('./profile')
 const helpers = require('./helpers')
@@ -369,6 +369,8 @@ const INSPECT_VALUE_DATE_TIME_MAPPING = {
   // and fill this mapping based on the decision
   '%d/%m/%y': 'date',
   '%d/%m/%Y': 'date',
+  '%m/%d/%y': 'date',
+  '%m/%d/%Y': 'date',
   '%H:%M': 'time',
 }
 const INSPECT_VALUE_GUESS_ORDER = [
@@ -414,7 +416,7 @@ function inspectValue(value) {
 
     // Guess date/time
     for (const [format, type] of Object.entries(INSPECT_VALUE_DATE_TIME_MAPPING)) {
-      if (timeParse(format)(value)) {
+      if (moment(value, helpers.convertDatetimeFormatFromFDtoJS(format), true).isValid()) {
         return { type, format }
       }
     }

diff --git a/src/types/date.js b/src/types/date.js
@@ -1,8 +1,8 @@
 const moment = require('moment')
 const isDate = require('lodash/isDate')
 const isString = require('lodash/isString')
-const { timeParse } = require('d3-time-format')
 const { ERROR } = require('../config')
+const helpers = require('../helpers')
 
 // Module API
 
@@ -29,9 +29,7 @@ function castDate(format, value) {
           )
           format = format.replace('fmt:', '')
         }
-        // https://github.com/d3/d3-time-format/issues/47
-        // It doesn't raise any error if the value is out-of-range
-        value = moment(timeParse(format)(value))
+        value = moment(value, helpers.convertDatetimeFormatFromFDtoJS(format), true)
       }
       if (!value.isValid()) {
         return ERROR

diff --git a/src/types/datetime.js b/src/types/datetime.js
@@ -1,8 +1,8 @@
 const moment = require('moment')
 const isDate = require('lodash/isDate')
 const isString = require('lodash/isString')
-const { timeParse } = require('d3-time-format')
 const { ERROR } = require('../config')
+const helpers = require('../helpers')
 
 // Module API
 
@@ -29,9 +29,7 @@ function castDatetime(format, value) {
           )
           format = format.replace('fmt:', '')
         }
-        // https://github.com/d3/d3-time-format/issues/47
-        // It doesn't raise any error if the value is out-of-range
-        value = moment(timeParse(format)(value))
+        value = moment(value, helpers.convertDatetimeFormatFromFDtoJS(format), true)
       }
       if (!value.isValid()) {
         return ERROR

diff --git a/src/types/time.js b/src/types/time.js
@@ -1,8 +1,8 @@
 const moment = require('moment')
 const isDate = require('lodash/isDate')
 const isString = require('lodash/isString')
-const { timeParse } = require('d3-time-format')
 const { ERROR } = require('../config')
+const helpers = require('../helpers')
 
 // Module API
 
@@ -13,7 +13,7 @@ function castTime(format, value) {
     }
     try {
       if (format === 'default') {
-        value = moment(timeParse(_DEFAULT_PATTERN)(value))
+        value = moment(value, _DEFAULT_PATTERN, true)
       } else if (format === 'any') {
         try {
           moment.suppressDeprecationWarnings = true
@@ -29,7 +29,7 @@ function castTime(format, value) {
           )
           format = format.replace('fmt:', '')
         }
-        value = moment(timeParse(format)(value))
+        value = moment(value, helpers.convertDatetimeFormatFromFDtoJS(format), true)
       }
       if (!value.isValid()) {
         return ERROR
@@ -48,4 +48,4 @@ module.exports = {
 
 // Internal
 
-const _DEFAULT_PATTERN = '%H:%M:%S'
+const _DEFAULT_PATTERN = 'HH:mm:ss'
diff --git a/test/infer.js b/test/infer.js
@@ -90,4 +90,16 @@ describe('infer', () => {
       { name: 'name', type: 'time', format: '%H:%M' },
     ])
   })
+
+  it('should infer "%m/%d/%Y" date formats (#134)', async () => {
+    const descriptor = await infer('data/data_infer_dates.csv')
+    assert.deepEqual(descriptor.fields, [
+      { format: 'default', name: 'first_name', type: 'string' },
+      { format: 'default', name: 'last_name', type: 'string' },
+      { format: 'email', name: 'email', type: 'string' },
+      { format: 'default', name: 'gender', type: 'string' },
+      { format: 'default', name: 'ip_address', type: 'string' },
+      { format: '%m/%d/%Y', name: 'date', type: 'date' },
+    ])
+  })
 })
diff --git a/test/types/date.js b/test/types/date.js
@@ -33,6 +33,8 @@ const TESTS = [
   ['invalid', '21/11/06 16:30', ERROR],
   ['default', '1999-11-31', ERROR],
   ['default', '1999-02-29', ERROR],
+  ['%Y-%m-%d', '1999-11-31', ERROR],
+  ['%Y-%m-%d', '1999-02-29', ERROR],
   ['default', '2000-02-29', date(2000, 2, 29)],
   // Deprecated
   ['fmt:%d/%m/%y', date(2019, 1, 1), date(2019, 1, 1)],

diff --git a/test/types/time.js b/test/types/time.js
@@ -1,11 +1,12 @@
+const moment = require('moment')
 const { assert } = require('chai')
 const { ERROR } = require('../../src/config')
 const types = require('../../src/types')
 
 // Helpers
 
 function time(hour, minute = 0, second = 0) {
-  return new Date(0, 0, 1, hour, minute, second)
+  return moment(`${hour}:${minute}:${second}`, 'h:m:s', true).toDate()
 }
 
 // Constants