Extend dictionary item schema (#302)

api/deps.edn

@@ -1,6 +1,6 @@
 {:deps      {accelerated-text/core                {:local/root "../core"}
              ch.qos.logback/logback-classic       {:mvn/version "1.2.3"}
-             com.walmartlabs/lacinia              {:mvn/version "0.35.0"}
+             com.walmartlabs/lacinia              {:mvn/version "0.38.0"}
              http-kit/http-kit                    {:mvn/version "2.3.0"}
              javax.servlet/servlet-api            {:mvn/version "2.5"}
              ring/ring-core                       {:mvn/version "1.7.1"}

api/resources/schema.graphql

@@ -11,30 +11,47 @@ enum Usage {
     DONT_CARE
 }
 
-#   Taken from: https://www.clips.uantwerpen.be/pages/mbsp-tags
+enum Language {
+    Eng
+    Ger
+    Est
+    Lit
+    Lav
+    Rus
+    Spa
+}
+
 enum PartOfSpeech {
-   A
-   A2
-   compoundA
-   Adv
-   N
-   N2
-   N3
-   PN
-   Prep
-   V
-   V0
-   V2
-   V3
-   V2A
-   V2Q
-   V2S
-   V2V
-   VA
-   VP
-   VQ
-   VS
-   VV
+    A
+    A2
+    AdA
+    AdN
+    AdV
+    Adv
+    Conj
+    IP
+    Interj
+    N
+    N2
+    N3
+    NP
+    PN
+    Post
+    Prep
+    Pron
+    Quant
+    Subj
+    V
+    V2
+    V2A
+    V2S
+    V2V
+    V2Q
+    V3
+    VA
+    VQ
+    VS
+    VV
 }
 
 ### Data -----------------------------------------------------------------------
@@ -143,12 +160,22 @@ type DictionaryResults {
     totalCount:         Int!
 }
 
+type DictionaryItemAttribute {
+    id:                 ID!
+    name:               String!
+    value:              String!
+}
+
 type DictionaryItem {
     id:                 ID!
     name:               String!
     partOfSpeech:       PartOfSpeech
     phrases:            [ Phrase! ]
     concept:            Concept
+    language:           Language
+    sense:              String
+    definition:         String
+    attributes:         [ DictionaryItemAttribute ]
 }
 
 type Phrase {
@@ -211,6 +238,7 @@ type DocumentPlan {
     dataSampleId:      ID
     dataSampleRow:     Int
     dataSampleMethod:  String
+    examples:          [ String ]
     createdAt:         Int!
     updatedAt:         Int
     updateCount:       Int!
@@ -324,20 +352,37 @@ type Query {
 
 ### Mutations ------------------------------------------------------------------
 
+input Attribute {
+    id:                     ID
+    name:                   String!
+    value:                  String!
+}
+
 type Mutation {
 
     ### Dictionary
 
     createDictionaryItem(
         id:                 ID
         name:               String!
-        partOfSpeech:       PartOfSpeech  # default = VB
+        partOfSpeech:       PartOfSpeech  # default = V
+        key:                String
+        forms:              [ String ]
+        language:           Language
+        sense:              String
+        definition:         String
+        attributes:         [ Attribute ]
     ):  DictionaryItem!
 
     updateDictionaryItem(
         id:                 ID!
         name:               String
         partOfSpeech:       PartOfSpeech
+        key:                String
+        language:           Language
+        sense:              String
+        definition:         String
+        attributes:         [ Attribute ]
     ):  DictionaryItem!
 
     deleteDictionaryItem(
@@ -363,6 +408,7 @@ type Mutation {
         dataSampleId:       ID
         dataSampleRow:      Int
         dataSampleMethod:   String
+        examples:           [ String ]
     ): DocumentPlan!
 
     deleteDocumentPlan(
@@ -379,6 +425,7 @@ type Mutation {
         dataSampleId:       ID
         dataSampleRow:      Int
         dataSampleMethod:   String
+        examples:           [ String ]
     ): DocumentPlan!
 
     createPhrase(

api/src/api/graphql/translate/concept.clj

@@ -1,5 +1,5 @@
 (ns api.graphql.translate.concept
-  (:require [acc-text.nlg.gf.paths :refer [possible-paths]]
+  (:require [acc-text.nlg.gf.paths :as paths]
             [acc-text.nlg.semantic-graph :as sg]
             [acc-text.nlg.semantic-graph.utils :as sg-utils]
             [clojure.string :as str]
@@ -9,7 +9,7 @@
   {:id         (utils/gen-rand-str 16)
    :fieldLabel (or name category "Str")
    :fieldType  (cond-> ["List" "Str"]
-                       (some? category) (-> (concat (get possible-paths category))
+                       (some? category) (-> (concat (get paths/possible-paths category))
                                             (distinct)
                                             (vec)
                                             (conj category)))})
@@ -22,12 +22,19 @@
    :label    label
    :name     (str/join " -> " (conj args category))})
 
+(defn find-categories [roles]
+  (reduce-kv (fn [m k v]
+               (assoc m k (paths/get-intersection (map (comp #(str/replace % #"[()]" "") :category) v))))
+             {}
+             (group-by :name roles)))
+
 (defn select-roles [roles]
-  (let [role-position (zipmap (distinct (map :name roles)) (range))]
+  (let [role-category (find-categories roles)
+        role-position (zipmap (distinct (map :name roles)) (range))]
     (->> roles
          (group-by :name)
          (vals)
-         (map (comp #(or % "Str") first reverse #(sort-by :category %)))
+         (map (comp #(assoc % :category (or (get role-category (:name %)) "Str")) first))
          (sort-by (comp role-position :name)))))
 
 (defn amr->schema [{::sg/keys [id category description name] :as entity}]

api/src/api/graphql/translate/dictionary.clj

@@ -1,6 +1,7 @@
 (ns api.graphql.translate.dictionary
   (:require [acc-text.nlg.dictionary.item :as dict-item]
             [acc-text.nlg.dictionary.item.form :as dict-item-form]
+            [acc-text.nlg.dictionary.item.attr :as dict-item-attr]
             [data.entities.reader-model :as reader-model]
             [data.utils :as utils]))
 
@@ -19,23 +20,42 @@
                   :defaultUsage (if enabled? "YES" "NO")}})
        (reader-model/available-languages)))
 
-(defn dictionary-item->schema [{::dict-item/keys [id key category forms language]}]
+(defn dictionary-item->schema [{::dict-item/keys [id key category forms language definition sense attributes]}]
   {:id           (or id (utils/gen-uuid))
    :name         key
    :partOfSpeech category
+   :language     language
+   :sense        sense
+   :definition   definition
    :phrases      (map (fn [{::dict-item-form/keys [id value default?]}]
                         {:id              id
                          :text            value
                          :defaultUsage    (if default? "YES" "NO")
                          :readerFlagUsage (build-reader-model-user-flags language)})
-                      forms)})
+                      forms)
+   :attributes   (map (fn [{::dict-item-attr/keys [id name value]}]
+                        {:id    id
+                         :name  name
+                         :value value})
+                      attributes)})
 
-(defn schema->dictionary-item [{id :id item-name :name pos :partOfSpeech}]
-  #::dict-item{:id       (or id (utils/gen-uuid))
-               :key      (if (some? pos)
-                           (format "%s_%s" item-name (name pos))
-                           item-name)
-               :category (name pos)
-               :sense    "1"
-               :language "Eng"
-               :forms    [#::dict-item-form{:id (utils/gen-uuid) :value item-name}]})
+(defn schema->dictionary-item
+  [{id :id item-name :name key :key pos :partOfSpeech forms :forms lang :language sense :sense definition :definition attrs :attributes}]
+  #::dict-item{:id         (or id (utils/gen-uuid))
+               :key        (cond
+                             (some? key) key
+                             (some? pos) (format "%s_%s" item-name (name pos))
+                             :else item-name)
+               :category   (if (some? pos) (name pos) "V")
+               :sense      (or sense "1")
+               :definition (or definition "")
+               :language   (if (some? lang) (name lang) "Eng")
+               :forms      (map (fn [form]
+                                  #::dict-item-form{:id    (utils/gen-uuid)
+                                                    :value form})
+                                (or (seq forms) [item-name]))
+               :attributes (map (fn [{:keys [id name value]}]
+                                  #::dict-item-attr{:id    (or id (utils/gen-uuid))
+                                                    :name  name
+                                                    :value value})
+                                attrs)})

api/src/api/graphql/translate/document_plan.clj

@@ -1,12 +1,14 @@
 (ns api.graphql.translate.document-plan
   (:require [api.utils :refer [read-mapper]]
+            [clojure.string :as str]
             [jsonista.core :as json]))
 
-(defn schema->dp [{:keys [id uid name kind blocklyXml documentPlan dataSampleId dataSampleRow dataSampleMethod]}]
+(defn schema->dp [{:keys [id uid name kind examples blocklyXml documentPlan dataSampleId dataSampleRow dataSampleMethod]}]
   {:id               id
    :uid              uid
    :name             name
    :kind             kind
+   :examples         (remove str/blank? examples)
    :blocklyXml       blocklyXml
    :documentPlan     (json/read-value documentPlan read-mapper)
    :dataSampleId     dataSampleId

api/src/api/nlg/parser.clj

@@ -394,7 +394,7 @@
 
 (defn select-kind [kinds]
   (cond
-    (= 1 (count kinds)) (first kinds)))
+    (= 1 (count (set kinds))) (first kinds)))
 
 (defn post-add-kind [{:keys [name type kind] :as node} {variables :variables}]
   (let [kinds (map :kind (remove definition? (dp-zip/get-children node)))

api/src/data/entities/document_plan/utils.clj

@@ -2,7 +2,8 @@
   (:require [data.entities.document-plan.zip :as dp-zip]
             [clojure.zip :as zip]
             [clojure.java.io :as io]
-            [clojure.data.xml :as xml]))
+            [clojure.data.xml :as xml]
+            [clojure.string :as str]))
 
 (defn get-variable-labels [blockly-xml]
   (when (some? blockly-xml)
@@ -24,12 +25,14 @@
        (map zip/node)
        (filter #(contains? types (:type %)))))
 
-(defn find-examples [{body :documentPlan blockly-xml :blocklyXml}]
+(defn find-examples [{body :documentPlan examples :examples blockly-xml :blocklyXml}]
   (let [labels (get-variable-labels blockly-xml)]
-    (->> (get-nodes-with-types body #{"Define-var"})
-         (filter #(and (= "Define-var" (:type %)) (= "*Description" (get labels (:name %)))))
-         (map (comp :text :value))
-         (remove nil?))))
+    (or (->> (get-nodes-with-types body #{"Define-var"})
+             (filter #(and (= "Define-var" (:type %)) (= "*Description" (get labels (:name %)))))
+             (map (comp :text :value))
+             (remove str/blank?)
+             (seq))
+        (remove str/blank? examples))))
 
 (defn find-variables [{body :documentPlan} labels]
   (->> (get-nodes-with-types body #{"Define-var"})

api/test/api/graphql/dictionary_test.clj

@@ -51,6 +51,27 @@
     (is (= "test_V" name))
     (is (= "V" partOfSpeech))))
 
+(deftest ^:integration create-complex-dict-item-test
+  (let [query "mutation CreateDictionaryItem($name: String!, $partOfSpeech: PartOfSpeech, $key: String, $forms: [String], $language: Language, $sense: String, $definition: String, $attributes: [Attribute]) { createDictionaryItem(name: $name, partOfSpeech: $partOfSpeech, key: $key, forms: $forms, language: $language, sense: $sense, definition: $definition, attributes: $attributes) { name partOfSpeech language sense definition phrases { id text } attributes { id name value } } } "
+        {{{{:keys [name partOfSpeech language sense definition phrases attributes]} :createDictionaryItem} :data errors :errors} :body}
+        (q "/_graphql" :post {:query query :variables {:name         "test"
+                                                       :partOfSpeech "N"
+                                                       :key          "test_key"
+                                                       :language     "Eng"
+                                                       :sense        "test"
+                                                       :definition   "test"
+                                                       :forms        ["test" "tests"]
+                                                       :attributes   [{:name  "Gender"
+                                                                       :value "nonhuman"}]}})]
+    (is (nil? errors))
+    (is (= "test_key" name))
+    (is (= "N" partOfSpeech))
+    (is (= language "Eng"))
+    (is (= sense "test"))
+    (is (= definition "test"))
+    (is (= ["test" "tests"] (map :text phrases)))
+    (is (= {"Gender" "nonhuman"} (into {} (map (fn [{:keys [name value]}] [name value]) attributes))))))
+
 (deftest ^:integration delete-dict-item-test
   (let [query "mutation DeleteDictionaryItem($id:ID!){deleteDictionaryItem(id:$id)}"
         {{{response :deleteDictionaryItem} :data errors :errors} :body}

core/src/acc_text/nlg/core.clj

@@ -1,5 +1,6 @@
 (ns acc-text.nlg.core
-  (:require [acc-text.nlg.dictionary.item :as dictionary-item]
+  (:require [acc-text.nlg.dictionary.item :as dict-item]
+            [acc-text.nlg.dictionary.item.attr :as dict-item-attr]
             [acc-text.nlg.semantic-graph :as sg]
             [acc-text.nlg.gf.service :as gf-service]
             [acc-text.nlg.grammar :as grammar]
@@ -12,9 +13,14 @@
       (update :amr #(zipmap (map ::sg/id %) %))
       (assoc :constants {"*Language" lang
                          "*Reader"   (str/join "," (:readers context))})
-      (update :dictionary #(zipmap (map (fn [{::dictionary-item/keys [key category]}]
-                                          [key category])
-                                        %)
+      (update :dictionary #(reduce (fn [m {::dict-item/keys [key category] :as dict-item}]
+                                     (assoc m [key category]
+                                              (update dict-item ::dict-item/attributes
+                                                      (fn [attrs]
+                                                        (into {} (map (fn [{::dict-item-attr/keys [name value]}]
+                                                                        [name value])
+                                                                      attrs))))))
+                                   {}
                                    %))))
 
 (defn generate-text [semantic-graph context lang]

core/src/acc_text/nlg/gf/paths.clj

@@ -1,6 +1,7 @@
 (ns acc-text.nlg.gf.paths
   (:require [acc-text.nlg.gf.utils :as utils]
-            [clojure.java.io :as io]))
+            [clojure.java.io :as io]
+            [clojure.set :as set]))
 
 (defn load-paths [resource-path]
   (utils/read-edn (io/file (io/resource resource-path))))
@@ -20,3 +21,10 @@
        (reduce-kv (fn [m k v]
                     (assoc m k (set (map first v))))
                   {})))
+
+(defn get-intersection [categories]
+  (->> categories
+       (map #(set (conj (get possible-paths %) %)))
+       (apply set/intersection)
+       (some #(when (contains? (set categories) %)
+                %))))