diff --git a/api/deps.edn b/api/deps.edn index 4f72c418b..e60902894 100644 --- a/api/deps.edn +++ b/api/deps.edn @@ -1,6 +1,6 @@ {:deps {accelerated-text/core {:local/root "../core"} ch.qos.logback/logback-classic {:mvn/version "1.2.3"} - com.walmartlabs/lacinia {:mvn/version "0.35.0"} + com.walmartlabs/lacinia {:mvn/version "0.38.0"} http-kit/http-kit {:mvn/version "2.3.0"} javax.servlet/servlet-api {:mvn/version "2.5"} ring/ring-core {:mvn/version "1.7.1"} diff --git a/api/resources/schema.graphql b/api/resources/schema.graphql index 10b8c6d8e..9fc7198f8 100644 --- a/api/resources/schema.graphql +++ b/api/resources/schema.graphql @@ -11,30 +11,47 @@ enum Usage { DONT_CARE } -# Taken from: https://www.clips.uantwerpen.be/pages/mbsp-tags +enum Language { + Eng + Ger + Est + Lit + Lav + Rus + Spa +} + enum PartOfSpeech { - A - A2 - compoundA - Adv - N - N2 - N3 - PN - Prep - V - V0 - V2 - V3 - V2A - V2Q - V2S - V2V - VA - VP - VQ - VS - VV + A + A2 + AdA + AdN + AdV + Adv + Conj + IP + Interj + N + N2 + N3 + NP + PN + Post + Prep + Pron + Quant + Subj + V + V2 + V2A + V2S + V2V + V2Q + V3 + VA + VQ + VS + VV } ### Data ----------------------------------------------------------------------- @@ -143,12 +160,22 @@ type DictionaryResults { totalCount: Int! } +type DictionaryItemAttribute { + id: ID! + name: String! + value: String! +} + type DictionaryItem { id: ID! name: String! partOfSpeech: PartOfSpeech phrases: [ Phrase! ] concept: Concept + language: Language + sense: String + definition: String + attributes: [ DictionaryItemAttribute ] } type Phrase { @@ -211,6 +238,7 @@ type DocumentPlan { dataSampleId: ID dataSampleRow: Int dataSampleMethod: String + examples: [ String ] createdAt: Int! updatedAt: Int updateCount: Int! @@ -324,6 +352,12 @@ type Query { ### Mutations ------------------------------------------------------------------ +input Attribute { + id: ID + name: String! + value: String! +} + type Mutation { ### Dictionary @@ -331,13 +365,24 @@ type Mutation { createDictionaryItem( id: ID name: String! - partOfSpeech: PartOfSpeech # default = VB + partOfSpeech: PartOfSpeech # default = V + key: String + forms: [ String ] + language: Language + sense: String + definition: String + attributes: [ Attribute ] ): DictionaryItem! updateDictionaryItem( id: ID! name: String partOfSpeech: PartOfSpeech + key: String + language: Language + sense: String + definition: String + attributes: [ Attribute ] ): DictionaryItem! deleteDictionaryItem( @@ -363,6 +408,7 @@ type Mutation { dataSampleId: ID dataSampleRow: Int dataSampleMethod: String + examples: [ String ] ): DocumentPlan! deleteDocumentPlan( @@ -379,6 +425,7 @@ type Mutation { dataSampleId: ID dataSampleRow: Int dataSampleMethod: String + examples: [ String ] ): DocumentPlan! createPhrase( diff --git a/api/src/api/graphql/translate/concept.clj b/api/src/api/graphql/translate/concept.clj index 667a54421..9ea97508c 100644 --- a/api/src/api/graphql/translate/concept.clj +++ b/api/src/api/graphql/translate/concept.clj @@ -1,5 +1,5 @@ (ns api.graphql.translate.concept - (:require [acc-text.nlg.gf.paths :refer [possible-paths]] + (:require [acc-text.nlg.gf.paths :as paths] [acc-text.nlg.semantic-graph :as sg] [acc-text.nlg.semantic-graph.utils :as sg-utils] [clojure.string :as str] @@ -9,7 +9,7 @@ {:id (utils/gen-rand-str 16) :fieldLabel (or name category "Str") :fieldType (cond-> ["List" "Str"] - (some? category) (-> (concat (get possible-paths category)) + (some? category) (-> (concat (get paths/possible-paths category)) (distinct) (vec) (conj category)))}) @@ -22,12 +22,19 @@ :label label :name (str/join " -> " (conj args category))}) +(defn find-categories [roles] + (reduce-kv (fn [m k v] + (assoc m k (paths/get-intersection (map (comp #(str/replace % #"[()]" "") :category) v)))) + {} + (group-by :name roles))) + (defn select-roles [roles] - (let [role-position (zipmap (distinct (map :name roles)) (range))] + (let [role-category (find-categories roles) + role-position (zipmap (distinct (map :name roles)) (range))] (->> roles (group-by :name) (vals) - (map (comp #(or % "Str") first reverse #(sort-by :category %))) + (map (comp #(assoc % :category (or (get role-category (:name %)) "Str")) first)) (sort-by (comp role-position :name))))) (defn amr->schema [{::sg/keys [id category description name] :as entity}] diff --git a/api/src/api/graphql/translate/dictionary.clj b/api/src/api/graphql/translate/dictionary.clj index a07137fa2..57ab9f4bf 100644 --- a/api/src/api/graphql/translate/dictionary.clj +++ b/api/src/api/graphql/translate/dictionary.clj @@ -1,6 +1,7 @@ (ns api.graphql.translate.dictionary (:require [acc-text.nlg.dictionary.item :as dict-item] [acc-text.nlg.dictionary.item.form :as dict-item-form] + [acc-text.nlg.dictionary.item.attr :as dict-item-attr] [data.entities.reader-model :as reader-model] [data.utils :as utils])) @@ -19,23 +20,42 @@ :defaultUsage (if enabled? "YES" "NO")}}) (reader-model/available-languages))) -(defn dictionary-item->schema [{::dict-item/keys [id key category forms language]}] +(defn dictionary-item->schema [{::dict-item/keys [id key category forms language definition sense attributes]}] {:id (or id (utils/gen-uuid)) :name key :partOfSpeech category + :language language + :sense sense + :definition definition :phrases (map (fn [{::dict-item-form/keys [id value default?]}] {:id id :text value :defaultUsage (if default? "YES" "NO") :readerFlagUsage (build-reader-model-user-flags language)}) - forms)}) + forms) + :attributes (map (fn [{::dict-item-attr/keys [id name value]}] + {:id id + :name name + :value value}) + attributes)}) -(defn schema->dictionary-item [{id :id item-name :name pos :partOfSpeech}] - #::dict-item{:id (or id (utils/gen-uuid)) - :key (if (some? pos) - (format "%s_%s" item-name (name pos)) - item-name) - :category (name pos) - :sense "1" - :language "Eng" - :forms [#::dict-item-form{:id (utils/gen-uuid) :value item-name}]}) +(defn schema->dictionary-item + [{id :id item-name :name key :key pos :partOfSpeech forms :forms lang :language sense :sense definition :definition attrs :attributes}] + #::dict-item{:id (or id (utils/gen-uuid)) + :key (cond + (some? key) key + (some? pos) (format "%s_%s" item-name (name pos)) + :else item-name) + :category (if (some? pos) (name pos) "V") + :sense (or sense "1") + :definition (or definition "") + :language (if (some? lang) (name lang) "Eng") + :forms (map (fn [form] + #::dict-item-form{:id (utils/gen-uuid) + :value form}) + (or (seq forms) [item-name])) + :attributes (map (fn [{:keys [id name value]}] + #::dict-item-attr{:id (or id (utils/gen-uuid)) + :name name + :value value}) + attrs)}) diff --git a/api/src/api/graphql/translate/document_plan.clj b/api/src/api/graphql/translate/document_plan.clj index 6880006e5..fb5f261dc 100644 --- a/api/src/api/graphql/translate/document_plan.clj +++ b/api/src/api/graphql/translate/document_plan.clj @@ -1,12 +1,14 @@ (ns api.graphql.translate.document-plan (:require [api.utils :refer [read-mapper]] + [clojure.string :as str] [jsonista.core :as json])) -(defn schema->dp [{:keys [id uid name kind blocklyXml documentPlan dataSampleId dataSampleRow dataSampleMethod]}] +(defn schema->dp [{:keys [id uid name kind examples blocklyXml documentPlan dataSampleId dataSampleRow dataSampleMethod]}] {:id id :uid uid :name name :kind kind + :examples (remove str/blank? examples) :blocklyXml blocklyXml :documentPlan (json/read-value documentPlan read-mapper) :dataSampleId dataSampleId diff --git a/api/src/api/nlg/parser.clj b/api/src/api/nlg/parser.clj index 6f92ce74b..ffbd772ec 100644 --- a/api/src/api/nlg/parser.clj +++ b/api/src/api/nlg/parser.clj @@ -394,7 +394,7 @@ (defn select-kind [kinds] (cond - (= 1 (count kinds)) (first kinds))) + (= 1 (count (set kinds))) (first kinds))) (defn post-add-kind [{:keys [name type kind] :as node} {variables :variables}] (let [kinds (map :kind (remove definition? (dp-zip/get-children node))) diff --git a/api/src/data/entities/document_plan/utils.clj b/api/src/data/entities/document_plan/utils.clj index 51e3ff027..354b3f347 100644 --- a/api/src/data/entities/document_plan/utils.clj +++ b/api/src/data/entities/document_plan/utils.clj @@ -2,7 +2,8 @@ (:require [data.entities.document-plan.zip :as dp-zip] [clojure.zip :as zip] [clojure.java.io :as io] - [clojure.data.xml :as xml])) + [clojure.data.xml :as xml] + [clojure.string :as str])) (defn get-variable-labels [blockly-xml] (when (some? blockly-xml) @@ -24,12 +25,14 @@ (map zip/node) (filter #(contains? types (:type %))))) -(defn find-examples [{body :documentPlan blockly-xml :blocklyXml}] +(defn find-examples [{body :documentPlan examples :examples blockly-xml :blocklyXml}] (let [labels (get-variable-labels blockly-xml)] - (->> (get-nodes-with-types body #{"Define-var"}) - (filter #(and (= "Define-var" (:type %)) (= "*Description" (get labels (:name %))))) - (map (comp :text :value)) - (remove nil?)))) + (or (->> (get-nodes-with-types body #{"Define-var"}) + (filter #(and (= "Define-var" (:type %)) (= "*Description" (get labels (:name %))))) + (map (comp :text :value)) + (remove str/blank?) + (seq)) + (remove str/blank? examples)))) (defn find-variables [{body :documentPlan} labels] (->> (get-nodes-with-types body #{"Define-var"}) diff --git a/api/test/api/graphql/dictionary_test.clj b/api/test/api/graphql/dictionary_test.clj index 02ff069dc..82a812976 100644 --- a/api/test/api/graphql/dictionary_test.clj +++ b/api/test/api/graphql/dictionary_test.clj @@ -51,6 +51,27 @@ (is (= "test_V" name)) (is (= "V" partOfSpeech)))) +(deftest ^:integration create-complex-dict-item-test + (let [query "mutation CreateDictionaryItem($name: String!, $partOfSpeech: PartOfSpeech, $key: String, $forms: [String], $language: Language, $sense: String, $definition: String, $attributes: [Attribute]) { createDictionaryItem(name: $name, partOfSpeech: $partOfSpeech, key: $key, forms: $forms, language: $language, sense: $sense, definition: $definition, attributes: $attributes) { name partOfSpeech language sense definition phrases { id text } attributes { id name value } } } " + {{{{:keys [name partOfSpeech language sense definition phrases attributes]} :createDictionaryItem} :data errors :errors} :body} + (q "/_graphql" :post {:query query :variables {:name "test" + :partOfSpeech "N" + :key "test_key" + :language "Eng" + :sense "test" + :definition "test" + :forms ["test" "tests"] + :attributes [{:name "Gender" + :value "nonhuman"}]}})] + (is (nil? errors)) + (is (= "test_key" name)) + (is (= "N" partOfSpeech)) + (is (= language "Eng")) + (is (= sense "test")) + (is (= definition "test")) + (is (= ["test" "tests"] (map :text phrases))) + (is (= {"Gender" "nonhuman"} (into {} (map (fn [{:keys [name value]}] [name value]) attributes)))))) + (deftest ^:integration delete-dict-item-test (let [query "mutation DeleteDictionaryItem($id:ID!){deleteDictionaryItem(id:$id)}" {{{response :deleteDictionaryItem} :data errors :errors} :body} diff --git a/core/src/acc_text/nlg/core.clj b/core/src/acc_text/nlg/core.clj index 2326e938f..b8905519e 100644 --- a/core/src/acc_text/nlg/core.clj +++ b/core/src/acc_text/nlg/core.clj @@ -1,5 +1,6 @@ (ns acc-text.nlg.core - (:require [acc-text.nlg.dictionary.item :as dictionary-item] + (:require [acc-text.nlg.dictionary.item :as dict-item] + [acc-text.nlg.dictionary.item.attr :as dict-item-attr] [acc-text.nlg.semantic-graph :as sg] [acc-text.nlg.gf.service :as gf-service] [acc-text.nlg.grammar :as grammar] @@ -12,9 +13,14 @@ (update :amr #(zipmap (map ::sg/id %) %)) (assoc :constants {"*Language" lang "*Reader" (str/join "," (:readers context))}) - (update :dictionary #(zipmap (map (fn [{::dictionary-item/keys [key category]}] - [key category]) - %) + (update :dictionary #(reduce (fn [m {::dict-item/keys [key category] :as dict-item}] + (assoc m [key category] + (update dict-item ::dict-item/attributes + (fn [attrs] + (into {} (map (fn [{::dict-item-attr/keys [name value]}] + [name value]) + attrs)))))) + {} %)))) (defn generate-text [semantic-graph context lang] diff --git a/core/src/acc_text/nlg/gf/paths.clj b/core/src/acc_text/nlg/gf/paths.clj index 7bb5bbde7..58cdcc0c8 100644 --- a/core/src/acc_text/nlg/gf/paths.clj +++ b/core/src/acc_text/nlg/gf/paths.clj @@ -1,6 +1,7 @@ (ns acc-text.nlg.gf.paths (:require [acc-text.nlg.gf.utils :as utils] - [clojure.java.io :as io])) + [clojure.java.io :as io] + [clojure.set :as set])) (defn load-paths [resource-path] (utils/read-edn (io/file (io/resource resource-path)))) @@ -20,3 +21,10 @@ (reduce-kv (fn [m k v] (assoc m k (set (map first v)))) {}))) + +(defn get-intersection [categories] + (->> categories + (map #(set (conj (get possible-paths %) %))) + (apply set/intersection) + (some #(when (contains? (set categories) %) + %)))) diff --git a/core/src/acc_text/nlg/graph/categories.clj b/core/src/acc_text/nlg/graph/categories.clj index 31b4c5cdf..cebd62670 100644 --- a/core/src/acc_text/nlg/graph/categories.clj +++ b/core/src/acc_text/nlg/graph/categories.clj @@ -1,5 +1,6 @@ (ns acc-text.nlg.graph.categories - (:require [acc-text.nlg.graph.utils :as utils] + (:require [acc-text.nlg.gf.paths :as paths] + [acc-text.nlg.graph.utils :as utils] [loom.alg :as alg] [loom.graph :as graph])) @@ -21,6 +22,8 @@ (case (count categories) 0 g 1 (set-category g node-id (first categories)) - (throw (Exception. (format "Ambiguous categories for id `%s`" node-id)))))) + (if-let [intersection (paths/get-intersection categories)] + (set-category g node-id intersection) + (throw (Exception. (format "Ambiguous categories for id `%s`" node-id))))))) g (alg/pre-traverse g (utils/find-root-id g))))