From c0e4c8c9264498b3e61c4b9a1e6a9858a7cc3326 Mon Sep 17 00:00:00 2001
From: Martin Heuschober <martin.heuschober@posteo.net>
Date: Wed, 2 Jan 2019 13:42:41 -0800
Subject: [PATCH] fix issue #249 (wrong parsing of yyyy-mm-dd dates in DE)
 (#282)

Summary:
- took regex from EN (which is working)
- added test cases from 2016-01-01 to 2018-12-31

- change non-breaking space to regular space
Pull Request resolved: https://github.com/facebook/duckling/pull/282

Reviewed By: chinmay87

Differential Revision: D13551882

Pulled By: patapizza

fbshipit-source-id: e254be1beaed21226741c55f243ac21eaa06d636
---
 Duckling/Ranking/Classifiers/DE_XX.hs | 25 +++++++++++++++----------
 Duckling/Time/DE/Corpus.hs            | 13 ++++++++++---
 Duckling/Time/DE/Rules.hs             |  7 ++++---
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/Duckling/Ranking/Classifiers/DE_XX.hs b/Duckling/Ranking/Classifiers/DE_XX.hs
index e132a5eae..e05005265 100644
--- a/Duckling/Ranking/Classifiers/DE_XX.hs
+++ b/Duckling/Ranking/Classifiers/DE_XX.hs
@@ -35,12 +35,11 @@ classifiers
                                likelihoods = HashMap.fromList [], n = 0}}),
        ("integer (numeric)",
         Classifier{okData =
-                     ClassData{prior = -0.8641616664392084, unseen = -4.852030263919617,
+                     ClassData{prior = -0.8840298696559336, unseen = -4.852030263919617,
                                likelihoods = HashMap.fromList [("", 0.0)], n = 126},
                    koData =
-                     ClassData{prior = -0.5471519788929076,
-                               unseen = -5.1647859739235145,
-                               likelihoods = HashMap.fromList [("", 0.0)], n = 173}}),
+                     ClassData{prior = -0.5329259707666566, unseen = -5.198497031265826,
+                               likelihoods = HashMap.fromList [("", 0.0)], n = 179}}),
        ("exactly <time-of-day>",
         Classifier{okData =
                      ClassData{prior = 0.0, unseen = -2.3025850929940455,
@@ -742,6 +741,13 @@ classifiers
                    koData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
                                likelihoods = HashMap.fromList [], n = 0}}),
+       ("yyyy-mm-dd",
+        Classifier{okData =
+                     ClassData{prior = 0.0, unseen = -1.3862943611198906,
+                               likelihoods = HashMap.fromList [("", 0.0)], n = 2},
+                   koData =
+                     ClassData{prior = -infinity, unseen = -0.6931471805599453,
+                               likelihoods = HashMap.fromList [], n = 0}}),
        ("year (latent)",
         Classifier{okData =
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
@@ -954,9 +960,9 @@ classifiers
                      ClassData{prior = -infinity, unseen = -0.6931471805599453,
                                likelihoods = HashMap.fromList [], n = 0},
                    koData =
-                     ClassData{prior = 0.0, unseen = -2.995732273553991,
+                     ClassData{prior = 0.0, unseen = -3.1780538303479458,
                                likelihoods = HashMap.fromList [("integer (numeric)", 0.0)],
-                               n = 18}}),
+                               n = 22}}),
        ("in|during the <part-of-day>",
         Classifier{okData =
                      ClassData{prior = -6.0624621816434854e-2,
@@ -1083,15 +1089,14 @@ classifiers
                                n = 17}}),
        ("year",
         Classifier{okData =
-                     ClassData{prior = -8.338160893905101e-2,
+                     ClassData{prior = -0.16034265007517937,
                                unseen = -3.2188758248682006,
                                likelihoods = HashMap.fromList [("integer (numeric)", 0.0)],
                                n = 23},
                    koData =
-                     ClassData{prior = -2.5257286443082556,
-                               unseen = -1.3862943611198906,
+                     ClassData{prior = -1.9095425048844386, unseen = -1.791759469228055,
                                likelihoods = HashMap.fromList [("integer (numeric)", 0.0)],
-                               n = 2}}),
+                               n = 4}}),
        ("last <day-of-week> of <time>",
         Classifier{okData =
                      ClassData{prior = 0.0, unseen = -2.3978952727983707,
diff --git a/Duckling/Time/DE/Corpus.hs b/Duckling/Time/DE/Corpus.hs
index 23ad818b7..41f628518 100644
--- a/Duckling/Time/DE/Corpus.hs
+++ b/Duckling/Time/DE/Corpus.hs
@@ -23,11 +23,14 @@ import Duckling.Time.Corpus
 import Duckling.Time.Types hiding (Month)
 import Duckling.TimeGrain.Types hiding (add)
 
+context :: Context
+context = testContext {locale = makeLocale DE Nothing}
+
 corpus :: Corpus
-corpus = (testContext {locale = makeLocale DE Nothing}, testOptions, allExamples)
+corpus = (context, testOptions, allExamples)
 
 negativeCorpus :: NegativeCorpus
-negativeCorpus = (testContext {locale = makeLocale DE Nothing}, testOptions, examples)
+negativeCorpus = (context, testOptions, examples)
   where
     examples =
       [ "ein Hotel"
@@ -631,7 +634,7 @@ allExamples = concat
   , examples (datetime (2013, 12, 10, 0, 0, 0) Day)
              [ "10.12."
              ]
-  , examples (datetimeInterval ((2013, 2, 12, 18, 30, 0), (2013, 2, 12, 19, 1, 0)) Minute)
+  , examples (datetimeInterval ((2013, 2, 12, 18, 30, 0), (2013, 2, 12, 19, 1, 0)) Minute)
              [ "18:30h - 19:00h"
              , "18:30h/19:00h"
              ]
@@ -666,4 +669,8 @@ allExamples = concat
   , examples (datetime (2013, 2, 12, 17, 10, 0) Minute)
              [ "17h10"
              ]
+  , examples (datetime (2018, 8, 31, 0, 0, 0) Day)
+             [ "2018-08-31"
+             , "2018-8-31"
+             ]
   ]
diff --git a/Duckling/Time/DE/Rules.hs b/Duckling/Time/DE/Rules.hs
index 34df3fb38..88645f3b1 100644
--- a/Duckling/Time/DE/Rules.hs
+++ b/Duckling/Time/DE/Rules.hs
@@ -11,7 +11,8 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.Time.DE.Rules
-  ( rules ) where
+  ( rules
+  ) where
 
 import Prelude
 import Data.Text (Text)
@@ -20,7 +21,7 @@ import qualified Data.Text as Text
 import Duckling.Dimensions.Types
 import Duckling.Numeral.Helpers (parseInt)
 import Duckling.Ordinal.Types (OrdinalData (..))
-import Duckling.Regex.Types
+import Duckling.Regex.Types (GroupMatch(..))
 import Duckling.Time.Helpers
 import Duckling.Time.Types (TimeData (..))
 import Duckling.Types
@@ -1050,7 +1051,7 @@ ruleYyyymmdd :: Rule
 ruleYyyymmdd = Rule
   { name = "yyyy-mm-dd"
   , pattern =
-    [ regex "(\\d{2,4})-(0?[1-9]|10|11|12)-([012]?[1-9]|10|20|30|31)"
+    [ regex "(\\d{2,4})-(1[0-2]|0?[1-9])-(3[01]|[12]\\d|0?[1-9])"
     ]
   , prod = \tokens -> case tokens of
       (Token RegexMatch (GroupMatch (m1:m2:m3:_)):_) -> do