From 39a581c0198053ae0d9fbf20ffd4acf23efe4e1b Mon Sep 17 00:00:00 2001 From: Rokas Ramanauskas Date: Fri, 19 Mar 2021 14:46:16 +0200 Subject: [PATCH] Bring back async (#300) --- api/src/api/config.clj | 21 ++++++------ api/src/api/nlg/service.clj | 50 ++++++++++++++++------------ api/src/api/nlg/service/request.clj | 2 ++ api/src/api/nlg/service/response.clj | 14 ++++++++ api/src/api/nlg/service/utils.clj | 22 +++++++++--- api/src/api/server.clj | 10 +++--- api/test/api/end_to_end_test.clj | 6 ++++ api/test/api/test_utils.clj | 15 +++++---- 8 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 api/src/api/nlg/service/response.clj diff --git a/api/src/api/config.clj b/api/src/api/config.clj index adbe27086..c0e8d6998 100644 --- a/api/src/api/config.clj +++ b/api/src/api/config.clj @@ -7,16 +7,17 @@ (map (comp str/capitalize str/trim) (str/split l #","))) (defn load-config [] - {:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001")) - :host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0") - :db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")] - (keyword db-implementation)) - :db-uri (System/getenv "DB_URI") - :enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng"))) - :enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") ""))) - :config-path (or (System/getenv "CONFIG_PATH") (io/resource "config")) - :dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary")) - :document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans")) + {:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001")) + :host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0") + :db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")] + (keyword db-implementation)) + :db-uri (System/getenv "DB_URI") + :display-error (Boolean/valueOf (System/getenv "DISPLAY_ERROR")) + :enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng"))) + :enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") ""))) + :config-path (or (System/getenv "CONFIG_PATH") (io/resource "config")) + :dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary")) + :document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans")) :relevant-items-limit (or (System/getenv "RELEVANT_ITEMS_MATRIX_LIMIT") 100)}) (defstate conf :start (load-config)) diff --git a/api/src/api/nlg/service.clj b/api/src/api/nlg/service.clj index fa6fd1f65..960abbc81 100644 --- a/api/src/api/nlg/service.clj +++ b/api/src/api/nlg/service.clj @@ -1,7 +1,7 @@ (ns api.nlg.service (:require [api.nlg.core :refer [generate-text]] - [api.nlg.format :refer [use-format with-default-format]] [api.nlg.service.request :as request] + [api.nlg.service.response :as response] [api.nlg.service.utils :as utils] [api.utils :refer [gen-uuid]] [clojure.spec.alpha :as s] @@ -16,7 +16,8 @@ ::request/dataId ::request/dataRow ::request/sampleMethod - ::request/readerFlagValues])) + ::request/readerFlagValues + ::request/async])) (s/def ::generate-request-bulk (s/keys :req-un [::request/dataRows] @@ -24,26 +25,38 @@ ::request/documentPlanName ::request/readerFlagValues])) +(s/def ::generate-response + (s/keys :req-un [::response/resultId] + :opt-un [::response/offset + ::response/totalCount + ::response/ready + ::response/updatedAt + ::response/variants])) + (s/def ::get-result (s/keys :opt-un [::request/format])) -(defn display-error? [] - (Boolean/valueOf (System/getenv "DISPLAY_ERROR"))) - (defn generate-request - [{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues :as request}] + [{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues async :async :as request :or {async true}}] (try (log/infof "Generate request with %s" (utils/request->text request)) (let [{row-index :dataSampleRow :as document-plan} (utils/get-document-plan request) - result-id (gen-uuid)] + result-id (gen-uuid) + body {:id result-id + :document-plan document-plan + :data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {}) + :reader-model (map reader-model/update! (utils/form-reader-model reader-model))}] (results/write #::result{:id result-id :status :pending}) - (results/write (generate-text {:id result-id - :document-plan document-plan - :data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {}) - :reader-model (map reader-model/update! (utils/form-reader-model reader-model))})) - {:status 200 - :body {:resultId result-id}}) + (if (true? async) + (do + (future (results/write (generate-text body))) + {:status 200 + :body {:resultId result-id}}) + (do + (results/write (generate-text body)) + {:status 200 + :body (utils/translate-result (results/fetch result-id) {:format "raw"})}))) (catch Exception e (utils/error-response e "Generate request failure")))) @@ -73,16 +86,9 @@ (defn get-result [{{{request-id :id} :path {result-format :format} :query} :parameters}] (try (log/infof "Result request with id `%s`" request-id) - (if-let [{::result/keys [rows status timestamp] :as result} (results/fetch request-id)] + (if-let [result (results/fetch request-id)] {:status 200 - :body {:offset 0 - :totalCount (count rows) - :ready (not= status :pending) - :updatedAt timestamp - :variants (cond - (some? result-format) (use-format result-format result) - (and (= :error status) (display-error?)) (use-format "error" result) - :else (with-default-format result))}} + :body (utils/translate-result result {:format result-format})} (do (log/warnf "Result with id `%s` not found" request-id) {:status 404})) diff --git a/api/src/api/nlg/service/request.clj b/api/src/api/nlg/service/request.clj index 86e2d8a0e..3a65744df 100644 --- a/api/src/api/nlg/service/request.clj +++ b/api/src/api/nlg/service/request.clj @@ -16,3 +16,5 @@ (s/def ::dataRows (s/map-of ::id ::dataRow)) (s/def ::readerFlagValues (s/map-of string? boolean?)) + +(s/def ::async boolean?) diff --git a/api/src/api/nlg/service/response.clj b/api/src/api/nlg/service/response.clj new file mode 100644 index 000000000..efc053184 --- /dev/null +++ b/api/src/api/nlg/service/response.clj @@ -0,0 +1,14 @@ +(ns api.nlg.service.response + (:require [clojure.spec.alpha :as s])) + +(s/def ::resultId string?) + +(s/def ::offset int?) + +(s/def ::totalCount int?) + +(s/def ::ready boolean?) + +(s/def ::updatedAt number?) + +(s/def ::variants sequential?) diff --git a/api/src/api/nlg/service/utils.clj b/api/src/api/nlg/service/utils.clj index e9c3f821c..5b17d829b 100644 --- a/api/src/api/nlg/service/utils.clj +++ b/api/src/api/nlg/service/utils.clj @@ -1,10 +1,24 @@ (ns api.nlg.service.utils - (:require [api.nlg.enrich.data :as data-enrich] + (:require [api.config :refer [conf]] + [api.nlg.enrich.data :as data-enrich] + [api.nlg.format :refer [use-format with-default-format]] [clojure.string :as str] [clojure.tools.logging :as log] [data.entities.data-files :as data-files] [data.entities.document-plan :as dp] - [data.entities.reader-model :as reader-model])) + [data.entities.reader-model :as reader-model] + [data.spec.result :as result])) + +(defn translate-result [{::result/keys [id status timestamp rows] :as result} {result-format :format}] + {:resultId id + :offset 0 + :totalCount (count rows) + :ready (not= status :pending) + :updatedAt timestamp + :variants (cond + (some? result-format) (use-format result-format result) + (and (= :error status) (:display-error conf)) (use-format "error" result) + :else (with-default-format result))}) (defn error-response ([exception] (error-response exception nil)) @@ -32,8 +46,8 @@ (log/infof "Sample Method: %s" sample-method) (when-not (str/blank? data-id) (if-let [{[{fields :fields}] :records filename :fileName} (case sample-method - "relevant" (data-files/fetch-most-relevant data-id index 20) - "first" (data-files/fetch data-id index 1))] + "relevant" (data-files/fetch-most-relevant data-id index 20) + "first" (data-files/fetch data-id index 1))] (cond->> (zipmap (map :fieldName fields) (map :value fields)) (data-enrich/enable-enrich?) (data-enrich/enrich filename)) (log/errorf "Data with id `%s` not found" data-id)))) diff --git a/api/src/api/server.clj b/api/src/api/server.clj index 4f2ddeedd..3a0d2b2a5 100755 --- a/api/src/api/server.clj +++ b/api/src/api/server.clj @@ -34,9 +34,9 @@ (let [main-deps {:gf (gf-service/ping)} other-deps {} color (cond - (some false? (vals main-deps)) :red - (some false? (vals other-deps)) :yellow - :else :green)] + (some false? (vals main-deps)) :red + (some false? (vals other-deps)) :yellow + :else :green)] {:status 200 :body {:color (name color) :services (merge main-deps other-deps)}})) (defn string-store [item] @@ -62,7 +62,7 @@ :summary "GraphQL endpoint"} :options cors-handler}] ["/nlg/" {:post {:parameters {:body ::service/generate-request} - :responses {200 {:body {:resultId string?}}} + :responses {200 {:body ::service/generate-response}} :summary "Registers document plan for generation" :coercion reitit.coercion.spec/coercion :middleware [muuntaja/format-request-middleware @@ -101,7 +101,7 @@ :handler (swagger/create-swagger-handler)}}] ["/health" {:get health}] ["/status" {:get {:responses {200 {:body {:color string? :services coll?}}} - :handler status}}]] + :handler status}}]] {:data { :muuntaja m/instance :middleware [swagger/swagger-feature diff --git a/api/test/api/end_to_end_test.clj b/api/test/api/end_to_end_test.clj index 9de925f46..d78ed20a3 100644 --- a/api/test/api/end_to_end_test.clj +++ b/api/test/api/end_to_end_test.clj @@ -10,6 +10,12 @@ (is (= {"test" []} (generate-text-bulk {:document-plan-name "Untitled plan" :data-rows {"test" {}}})))) +(deftest ^:integration non-async-text-generation + (is (= ["Building Search Applications is authored by Manu Konchady."] + (:variants (generate-text {:document-plan-name "Authorship" + :data-file-name "books.csv" + :async false}))))) + (deftest ^:integration text-generation-with-simple-plan (is (= ["Building Search Applications is authored by Manu Konchady."] (generate-text {:document-plan-name "Authorship" diff --git a/api/test/api/test_utils.clj b/api/test/api/test_utils.clj index a5a9a6a70..c45a0e7bb 100644 --- a/api/test/api/test_utils.clj +++ b/api/test/api/test_utils.clj @@ -69,14 +69,15 @@ (< 10 retry-count) (throw (Exception. "Result was not ready after 10 seconds.")) :else (do (Thread/sleep 1000) (recur (inc retry-count) (request!)))))))) -(defn generate-text [{:keys [document-plan-name data-file-name reader-flags]}] - (let [{:keys [status body]} (q "/nlg/" :post {:documentPlanId (load-document-plan document-plan-name) - :readerFlagValues (or reader-flags {"Eng" true}) - :dataId (if (some? data-file-name) - (load-data-file data-file-name) - "")})] +(defn generate-text [{:keys [document-plan-name data-file-name reader-flags async]}] + (let [{:keys [status body]} (q "/nlg/" :post (cond-> {:documentPlanId (load-document-plan document-plan-name) + :readerFlagValues (or reader-flags {"Eng" true}) + :dataId (if (some? data-file-name) + (load-data-file data-file-name) + "")} + (false? async) (assoc :async false)))] (when (= 200 status) - (get-result (:resultId body))))) + (if (false? async) body (get-result (:resultId body)))))) (defn generate-text-bulk [{:keys [document-plan-name data-rows reader-flags]}] (let [{:keys [status body]} (q "/nlg/_bulk/" :post {:documentPlanId (load-document-plan document-plan-name)