Skip to content

Commit

Permalink
Bring back async (#300)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rokasramas
rokasramas committed Mar 19, 2021
1 parent 4becb30 commit 39a581c
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 48 deletions.
21 changes: 11 additions & 10 deletions api/src/api/config.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@
(map (comp str/capitalize str/trim) (str/split l #",")))

(defn load-config []
{:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
:host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
:db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
(keyword db-implementation))
:db-uri (System/getenv "DB_URI")
:enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
:enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
:config-path (or (System/getenv "CONFIG_PATH") (io/resource "config"))
:dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
:document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
{:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
:host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
:db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
(keyword db-implementation))
:db-uri (System/getenv "DB_URI")
:display-error (Boolean/valueOf (System/getenv "DISPLAY_ERROR"))
:enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
:enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
:config-path (or (System/getenv "CONFIG_PATH") (io/resource "config"))
:dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
:document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
:relevant-items-limit (or (System/getenv "RELEVANT_ITEMS_MATRIX_LIMIT") 100)})

(defstate conf :start (load-config))
50 changes: 28 additions & 22 deletions api/src/api/nlg/service.clj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
(ns api.nlg.service
(:require [api.nlg.core :refer [generate-text]]
[api.nlg.format :refer [use-format with-default-format]]
[api.nlg.service.request :as request]
[api.nlg.service.response :as response]
[api.nlg.service.utils :as utils]
[api.utils :refer [gen-uuid]]
[clojure.spec.alpha :as s]
Expand All @@ -16,34 +16,47 @@
::request/dataId
::request/dataRow
::request/sampleMethod
::request/readerFlagValues]))
::request/readerFlagValues
::request/async]))

(s/def ::generate-request-bulk
(s/keys :req-un [::request/dataRows]
:opt-un [::request/documentPlanId
::request/documentPlanName
::request/readerFlagValues]))

(s/def ::generate-response
(s/keys :req-un [::response/resultId]
:opt-un [::response/offset
::response/totalCount
::response/ready
::response/updatedAt
::response/variants]))

(s/def ::get-result
(s/keys :opt-un [::request/format]))

(defn display-error? []
(Boolean/valueOf (System/getenv "DISPLAY_ERROR")))

(defn generate-request
[{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues :as request}]
[{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues async :async :as request :or {async true}}]
(try
(log/infof "Generate request with %s" (utils/request->text request))
(let [{row-index :dataSampleRow :as document-plan} (utils/get-document-plan request)
result-id (gen-uuid)]
result-id (gen-uuid)
body {:id result-id
:document-plan document-plan
:data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
:reader-model (map reader-model/update! (utils/form-reader-model reader-model))}]
(results/write #::result{:id result-id
:status :pending})
(results/write (generate-text {:id result-id
:document-plan document-plan
:data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
:reader-model (map reader-model/update! (utils/form-reader-model reader-model))}))
{:status 200
:body {:resultId result-id}})
(if (true? async)
(do
(future (results/write (generate-text body)))
{:status 200
:body {:resultId result-id}})
(do
(results/write (generate-text body))
{:status 200
:body (utils/translate-result (results/fetch result-id) {:format "raw"})})))
(catch Exception e
(utils/error-response e "Generate request failure"))))

Expand Down Expand Up @@ -73,16 +86,9 @@
(defn get-result [{{{request-id :id} :path {result-format :format} :query} :parameters}]
(try
(log/infof "Result request with id `%s`" request-id)
(if-let [{::result/keys [rows status timestamp] :as result} (results/fetch request-id)]
(if-let [result (results/fetch request-id)]
{:status 200
:body {:offset 0
:totalCount (count rows)
:ready (not= status :pending)
:updatedAt timestamp
:variants (cond
(some? result-format) (use-format result-format result)
(and (= :error status) (display-error?)) (use-format "error" result)
:else (with-default-format result))}}
:body (utils/translate-result result {:format result-format})}
(do
(log/warnf "Result with id `%s` not found" request-id)
{:status 404}))
Expand Down
2 changes: 2 additions & 0 deletions api/src/api/nlg/service/request.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,5 @@
(s/def ::dataRows (s/map-of ::id ::dataRow))

(s/def ::readerFlagValues (s/map-of string? boolean?))

(s/def ::async boolean?)
14 changes: 14 additions & 0 deletions api/src/api/nlg/service/response.clj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
(ns api.nlg.service.response
(:require [clojure.spec.alpha :as s]))

(s/def ::resultId string?)

(s/def ::offset int?)

(s/def ::totalCount int?)

(s/def ::ready boolean?)

(s/def ::updatedAt number?)

(s/def ::variants sequential?)
22 changes: 18 additions & 4 deletions api/src/api/nlg/service/utils.clj
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
(ns api.nlg.service.utils
(:require [api.nlg.enrich.data :as data-enrich]
(:require [api.config :refer [conf]]
[api.nlg.enrich.data :as data-enrich]
[api.nlg.format :refer [use-format with-default-format]]
[clojure.string :as str]
[clojure.tools.logging :as log]
[data.entities.data-files :as data-files]
[data.entities.document-plan :as dp]
[data.entities.reader-model :as reader-model]))
[data.entities.reader-model :as reader-model]
[data.spec.result :as result]))

(defn translate-result [{::result/keys [id status timestamp rows] :as result} {result-format :format}]
{:resultId id
:offset 0
:totalCount (count rows)
:ready (not= status :pending)
:updatedAt timestamp
:variants (cond
(some? result-format) (use-format result-format result)
(and (= :error status) (:display-error conf)) (use-format "error" result)
:else (with-default-format result))})

(defn error-response
([exception] (error-response exception nil))
Expand Down Expand Up @@ -32,8 +46,8 @@
(log/infof "Sample Method: %s" sample-method)
(when-not (str/blank? data-id)
(if-let [{[{fields :fields}] :records filename :fileName} (case sample-method
"relevant" (data-files/fetch-most-relevant data-id index 20)
"first" (data-files/fetch data-id index 1))]
"relevant" (data-files/fetch-most-relevant data-id index 20)
"first" (data-files/fetch data-id index 1))]
(cond->> (zipmap (map :fieldName fields) (map :value fields))
(data-enrich/enable-enrich?) (data-enrich/enrich filename))
(log/errorf "Data with id `%s` not found" data-id))))
Expand Down
10 changes: 5 additions & 5 deletions api/src/api/server.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@
(let [main-deps {:gf (gf-service/ping)}
other-deps {}
color (cond
(some false? (vals main-deps)) :red
(some false? (vals other-deps)) :yellow
:else :green)]
(some false? (vals main-deps)) :red
(some false? (vals other-deps)) :yellow
:else :green)]
{:status 200 :body {:color (name color) :services (merge main-deps other-deps)}}))

(defn string-store [item]
Expand All @@ -62,7 +62,7 @@
:summary "GraphQL endpoint"}
:options cors-handler}]
["/nlg/" {:post {:parameters {:body ::service/generate-request}
:responses {200 {:body {:resultId string?}}}
:responses {200 {:body ::service/generate-response}}
:summary "Registers document plan for generation"
:coercion reitit.coercion.spec/coercion
:middleware [muuntaja/format-request-middleware
Expand Down Expand Up @@ -101,7 +101,7 @@
:handler (swagger/create-swagger-handler)}}]
["/health" {:get health}]
["/status" {:get {:responses {200 {:body {:color string? :services coll?}}}
:handler status}}]]
:handler status}}]]
{:data {
:muuntaja m/instance
:middleware [swagger/swagger-feature
Expand Down
6 changes: 6 additions & 0 deletions api/test/api/end_to_end_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@
(is (= {"test" []} (generate-text-bulk {:document-plan-name "Untitled plan"
:data-rows {"test" {}}}))))

(deftest ^:integration non-async-text-generation
(is (= ["Building Search Applications is authored by Manu Konchady."]
(:variants (generate-text {:document-plan-name "Authorship"
:data-file-name "books.csv"
:async false})))))

(deftest ^:integration text-generation-with-simple-plan
(is (= ["Building Search Applications is authored by Manu Konchady."]
(generate-text {:document-plan-name "Authorship"
Expand Down
15 changes: 8 additions & 7 deletions api/test/api/test_utils.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,15 @@
(< 10 retry-count) (throw (Exception. "Result was not ready after 10 seconds."))
:else (do (Thread/sleep 1000) (recur (inc retry-count) (request!))))))))

(defn generate-text [{:keys [document-plan-name data-file-name reader-flags]}]
(let [{:keys [status body]} (q "/nlg/" :post {:documentPlanId (load-document-plan document-plan-name)
:readerFlagValues (or reader-flags {"Eng" true})
:dataId (if (some? data-file-name)
(load-data-file data-file-name)
"")})]
(defn generate-text [{:keys [document-plan-name data-file-name reader-flags async]}]
(let [{:keys [status body]} (q "/nlg/" :post (cond-> {:documentPlanId (load-document-plan document-plan-name)
:readerFlagValues (or reader-flags {"Eng" true})
:dataId (if (some? data-file-name)
(load-data-file data-file-name)
"")}
(false? async) (assoc :async false)))]
(when (= 200 status)
(get-result (:resultId body)))))
(if (false? async) body (get-result (:resultId body))))))

(defn generate-text-bulk [{:keys [document-plan-name data-rows reader-flags]}]
(let [{:keys [status body]} (q "/nlg/_bulk/" :post {:documentPlanId (load-document-plan document-plan-name)
Expand Down

0 comments on commit 39a581c

Please sign in to comment.