Locales support

Summary: * Locales support for the library, following `<Lang>_<Region>` with ISO 639-1 code for `<Lang>` and ISO 3166-1 alpha-2 code for `<Region>` (#33) * `Locale` opaque type (composite of `Lang` and `Region`) with `makeLocale` smart constructor to only allow valid `(Lang, Region)` combinations * API: `Context`'s `lang` parameter has been replaced by `locale`, with optional `Region` and backward compatibility. * `Rules/<Lang>.hs` exposes - `langRules`: cross-locale rules for `<Lang>`, from `<Dimension>/<Lang>/Rules.hs` - `localeRules`: locale-specific rules, from `<Dimension>/<Lang>/<Region>/Rules.hs` - `defaultRules`: `langRules` + specific rules from select locales to ensure backward-compatibility * Corpus, tests & classifiers - 1 classifier per locale, with default classifier (`<Lang>_XX`) when no locale provided (backward-compatible) - Default classifiers are built on existing corpus - Locale classifiers are built on - `<Dimension>/<Lang>/Corpus.hs` exposes a common `corpus` to all locales of `<Lang>` - `<Dimension>/<Lang>/<Region>/Corpus.hs` exposes `allExamples`: a list of examples specific to the locale (following `<Dimension>/<Lang>/<Region>/Rules.hs`). - Locale classifiers use the language corpus extended with the locale examples as training set. - Locale examples need to use the same `Context` (i.e. reference time) as the language corpus. - For backward compatibility, `<Dimension>/<Lang>/Corpus.hs` can expose also `defaultCorpus`, which is `corpus` augmented with specific examples. This is controlled by `getDefaultCorpusForLang` in `Duckling.Ranking.Generate`. - Tests run against each classifier to make sure runtime works as expected. * MM/DD (en_US) vs DD/MM (en_GB) example to illustrate Reviewed By: JonCoens, blandinw Differential Revision: D6038096 fbshipit-source-id: f29c28d
facebook · Oct 13, 2017 · ab0ad02 · ab0ad02
1 parent ddefc94
commit ab0ad02
Show file tree

Hide file tree

Showing 247 changed files with 6,931 additions and 1,451 deletions.
diff --git a/Duckling/AmountOfMoney/BG/Corpus.hs b/Duckling/AmountOfMoney/BG/Corpus.hs
@@ -8,18 +8,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.BG.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
-import Duckling.Lang
 import Duckling.AmountOfMoney.Types
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = BG}, allExamples)
+corpus = (testContext {locale = makeLocale BG Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/EN/Corpus.hs b/Duckling/AmountOfMoney/EN/Corpus.hs
@@ -9,10 +9,11 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.EN.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
 import Duckling.Testing.Types

diff --git a/Duckling/AmountOfMoney/ES/Corpus.hs b/Duckling/AmountOfMoney/ES/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.ES.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = ES}, allExamples)
+corpus = (testContext {locale = makeLocale ES Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/FR/Corpus.hs b/Duckling/AmountOfMoney/FR/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.FR.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = FR}, allExamples)
+corpus = (testContext {locale = makeLocale FR Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/GA/Corpus.hs b/Duckling/AmountOfMoney/GA/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.GA.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = GA}, allExamples)
+corpus = (testContext {locale = makeLocale GA Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/HR/Corpus.hs b/Duckling/AmountOfMoney/HR/Corpus.hs
@@ -8,18 +8,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.HR.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
-import Duckling.Lang
 import Duckling.AmountOfMoney.Types
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = HR}, allExamples)
+corpus = (testContext {locale = makeLocale HR Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/ID/Corpus.hs b/Duckling/AmountOfMoney/ID/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.ID.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = ID}, allExamples)
+corpus = (testContext {locale = makeLocale ID Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/KO/Corpus.hs b/Duckling/AmountOfMoney/KO/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.KO.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = KO}, allExamples)
+corpus = (testContext {locale = makeLocale KO Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/NB/Corpus.hs b/Duckling/AmountOfMoney/NB/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.NB.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = NB}, allExamples)
+corpus = (testContext {locale = makeLocale NB Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/PT/Corpus.hs b/Duckling/AmountOfMoney/PT/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.PT.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = PT}, allExamples)
+corpus = (testContext {locale = makeLocale PT Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/RO/Corpus.hs b/Duckling/AmountOfMoney/RO/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.RO.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = RO}, allExamples)
+corpus = (testContext {locale = makeLocale RO Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/SV/Corpus.hs b/Duckling/AmountOfMoney/SV/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.SV.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = SV}, allExamples)
+corpus = (testContext {locale = makeLocale SV Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/AmountOfMoney/VI/Corpus.hs b/Duckling/AmountOfMoney/VI/Corpus.hs
@@ -9,18 +9,19 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 module Duckling.AmountOfMoney.VI.Corpus
-  ( corpus ) where
+  ( corpus
+  ) where
 
-import Prelude
 import Data.String
+import Prelude
 
 import Duckling.AmountOfMoney.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Testing.Types
 
 corpus :: Corpus
-corpus = (testContext {lang = VI}, allExamples)
+corpus = (testContext {locale = makeLocale VI Nothing}, allExamples)
 
 allExamples :: [Example]
 allExamples = concat

diff --git a/Duckling/Api.hs b/Duckling/Api.hs
@@ -17,18 +17,18 @@ module Duckling.Api
   ) where
 
 import Data.HashMap.Strict (HashMap)
-import qualified Data.HashMap.Strict as HashMap
 import Data.HashSet (HashSet)
-import qualified Data.HashSet as HashSet
 import Data.Text (Text)
-import qualified Data.Text as Text
 import Prelude
 import TextShow
+import qualified Data.HashMap.Strict as HashMap
+import qualified Data.HashSet as HashSet
+import qualified Data.Text as Text
 
 import Duckling.Dimensions.Types
 import Duckling.Dimensions
 import Duckling.Engine
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Ranking.Classifiers
 import Duckling.Ranking.Rank
 import Duckling.Resolve
@@ -47,11 +47,11 @@ supportedDimensions =
 -- When `targets` is non-empty, returns only tokens of such dimensions.
 analyze :: Text -> Context -> HashSet (Some Dimension) -> [ResolvedToken]
 analyze input context@Context{..} targets =
-  rank (classifiers lang) targets
+  rank (classifiers locale) targets
   . filter (\(Resolved{node = Node{token = (Token d _)}}) ->
       HashSet.null targets || HashSet.member (This d) targets
     )
-  $ parseAndResolve (rulesFor lang targets) input context
+  $ parseAndResolve (rulesFor locale targets) input context
 
 -- | Converts the resolved token to the API format
 formatToken :: Text -> ResolvedToken -> Entity

diff --git a/Duckling/Core.hs b/Duckling/Core.hs
@@ -12,16 +12,20 @@
 
 module Duckling.Core
   ( Context(..)
+  , Region(..)
   , Dimension(..)
-  , fromName
   , Entity(..)
   , Lang(..)
+  , Locale
   , Some(..)
+  , fromName
+  , makeLocale
   , toName
 
   -- Duckling API
   , parse
   , supportedDimensions
+  , allLocales
 
   -- Reference time builders
   , currentReftime
@@ -39,7 +43,7 @@ import qualified Data.HashMap.Strict as HashMap
 
 import Duckling.Api
 import Duckling.Dimensions.Types
-import Duckling.Lang
+import Duckling.Locale
 import Duckling.Resolve
 import Duckling.Types