Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bring back async #300

Merged
merged 4 commits into from
Mar 19, 2021
Merged

Bring back async #300

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
14e2032
feat: async generation
rokasramas Mar 10, 2021
47153a1
fix: generate text test util
rokasramas Mar 10, 2021
49b5b4a
refactor: return result with variants when async is false
rokasramas Mar 19, 2021
94c886b
Merge branch 'master' of github.com:tokenmill/accelerated-text into b…
rokasramas Mar 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 11 additions & 10 deletions api/src/api/config.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@
(map (comp str/capitalize str/trim) (str/split l #",")))

(defn load-config []
{:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
:host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
:db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
(keyword db-implementation))
:db-uri (System/getenv "DB_URI")
:enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
:enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
:config-path (or (System/getenv "CONFIG_PATH") (io/resource "config"))
:dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
:document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
{:port (Integer/valueOf ^String (or (System/getenv "ACC_TEXT_API_PORT") "3001"))
:host (or (System/getenv "ACC_TEXT_API_HOST") "0.0.0.0")
:db-implementation (when-let [db-implementation (or (System/getenv "DB_IMPLEMENTATION") "datomic")]
(keyword db-implementation))
:db-uri (System/getenv "DB_URI")
:display-error (Boolean/valueOf (System/getenv "DISPLAY_ERROR"))
:enabled-languages (set (read-conf-line (or (System/getenv "ENABLED_LANGUAGES") "Eng")))
:enabled-readers (set (read-conf-line (or (System/getenv "ENABLED_READERS") "")))
:config-path (or (System/getenv "CONFIG_PATH") (io/resource "config"))
:dictionary-path (or (System/getenv "DICT_PATH") (io/resource "dictionary"))
:document-plan-path (or (System/getenv "DOCUMENT_PLANS") (io/resource "document-plans"))
:relevant-items-limit (or (System/getenv "RELEVANT_ITEMS_MATRIX_LIMIT") 100)})

(defstate conf :start (load-config))
50 changes: 28 additions & 22 deletions api/src/api/nlg/service.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
(ns api.nlg.service
(:require [api.nlg.core :refer [generate-text]]
[api.nlg.format :refer [use-format with-default-format]]
[api.nlg.service.request :as request]
[api.nlg.service.response :as response]
[api.nlg.service.utils :as utils]
[api.utils :refer [gen-uuid]]
[clojure.spec.alpha :as s]
Expand All @@ -16,34 +16,47 @@
::request/dataId
::request/dataRow
::request/sampleMethod
::request/readerFlagValues]))
::request/readerFlagValues
::request/async]))

(s/def ::generate-request-bulk
(s/keys :req-un [::request/dataRows]
:opt-un [::request/documentPlanId
::request/documentPlanName
::request/readerFlagValues]))

(s/def ::generate-response
(s/keys :req-un [::response/resultId]
:opt-un [::response/offset
::response/totalCount
::response/ready
::response/updatedAt
::response/variants]))

(s/def ::get-result
(s/keys :opt-un [::request/format]))

(defn display-error? []
(Boolean/valueOf (System/getenv "DISPLAY_ERROR")))

(defn generate-request
[{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues :as request}]
[{data-id :dataId sample-method :sampleMethod data-row :dataRow reader-model :readerFlagValues async :async :as request :or {async true}}]
(try
(log/infof "Generate request with %s" (utils/request->text request))
(let [{row-index :dataSampleRow :as document-plan} (utils/get-document-plan request)
result-id (gen-uuid)]
result-id (gen-uuid)
body {:id result-id
:document-plan document-plan
:data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
:reader-model (map reader-model/update! (utils/form-reader-model reader-model))}]
(results/write #::result{:id result-id
:status :pending})
(results/write (generate-text {:id result-id
:document-plan document-plan
:data (or data-row (utils/get-data-row data-id (or sample-method "first") (or row-index 0)) {})
:reader-model (map reader-model/update! (utils/form-reader-model reader-model))}))
{:status 200
:body {:resultId result-id}})
(if (true? async)
(do
(future (results/write (generate-text body)))
{:status 200
:body {:resultId result-id}})
(do
(results/write (generate-text body))
{:status 200
:body (utils/translate-result (results/fetch result-id) {:format "raw"})})))
(catch Exception e
(utils/error-response e "Generate request failure"))))

Expand Down Expand Up @@ -73,16 +86,9 @@
(defn get-result [{{{request-id :id} :path {result-format :format} :query} :parameters}]
(try
(log/infof "Result request with id `%s`" request-id)
(if-let [{::result/keys [rows status timestamp] :as result} (results/fetch request-id)]
(if-let [result (results/fetch request-id)]
{:status 200
:body {:offset 0
:totalCount (count rows)
:ready (not= status :pending)
:updatedAt timestamp
:variants (cond
(some? result-format) (use-format result-format result)
(and (= :error status) (display-error?)) (use-format "error" result)
:else (with-default-format result))}}
:body (utils/translate-result result {:format result-format})}
(do
(log/warnf "Result with id `%s` not found" request-id)
{:status 404}))
Expand Down
2 changes: 2 additions & 0 deletions api/src/api/nlg/service/request.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,5 @@
(s/def ::dataRows (s/map-of ::id ::dataRow))

(s/def ::readerFlagValues (s/map-of string? boolean?))

(s/def ::async boolean?)
14 changes: 14 additions & 0 deletions api/src/api/nlg/service/response.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
(ns api.nlg.service.response
(:require [clojure.spec.alpha :as s]))

(s/def ::resultId string?)

(s/def ::offset int?)

(s/def ::totalCount int?)

(s/def ::ready boolean?)

(s/def ::updatedAt number?)

(s/def ::variants sequential?)
22 changes: 18 additions & 4 deletions api/src/api/nlg/service/utils.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
(ns api.nlg.service.utils
(:require [api.nlg.enrich.data :as data-enrich]
(:require [api.config :refer [conf]]
[api.nlg.enrich.data :as data-enrich]
[api.nlg.format :refer [use-format with-default-format]]
[clojure.string :as str]
[clojure.tools.logging :as log]
[data.entities.data-files :as data-files]
[data.entities.document-plan :as dp]
[data.entities.reader-model :as reader-model]))
[data.entities.reader-model :as reader-model]
[data.spec.result :as result]))

(defn translate-result [{::result/keys [id status timestamp rows] :as result} {result-format :format}]
{:resultId id
:offset 0
:totalCount (count rows)
:ready (not= status :pending)
:updatedAt timestamp
:variants (cond
(some? result-format) (use-format result-format result)
(and (= :error status) (:display-error conf)) (use-format "error" result)
:else (with-default-format result))})

(defn error-response
([exception] (error-response exception nil))
Expand Down Expand Up @@ -32,8 +46,8 @@
(log/infof "Sample Method: %s" sample-method)
(when-not (str/blank? data-id)
(if-let [{[{fields :fields}] :records filename :fileName} (case sample-method
"relevant" (data-files/fetch-most-relevant data-id index 20)
"first" (data-files/fetch data-id index 1))]
"relevant" (data-files/fetch-most-relevant data-id index 20)
"first" (data-files/fetch data-id index 1))]
(cond->> (zipmap (map :fieldName fields) (map :value fields))
(data-enrich/enable-enrich?) (data-enrich/enrich filename))
(log/errorf "Data with id `%s` not found" data-id))))
Expand Down
10 changes: 5 additions & 5 deletions api/src/api/server.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@
(let [main-deps {:gf (gf-service/ping)}
other-deps {}
color (cond
(some false? (vals main-deps)) :red
(some false? (vals other-deps)) :yellow
:else :green)]
(some false? (vals main-deps)) :red
(some false? (vals other-deps)) :yellow
:else :green)]
{:status 200 :body {:color (name color) :services (merge main-deps other-deps)}}))

(defn string-store [item]
Expand All @@ -62,7 +62,7 @@
:summary "GraphQL endpoint"}
:options cors-handler}]
["/nlg/" {:post {:parameters {:body ::service/generate-request}
:responses {200 {:body {:resultId string?}}}
:responses {200 {:body ::service/generate-response}}
:summary "Registers document plan for generation"
:coercion reitit.coercion.spec/coercion
:middleware [muuntaja/format-request-middleware
Expand Down Expand Up @@ -101,7 +101,7 @@
:handler (swagger/create-swagger-handler)}}]
["/health" {:get health}]
["/status" {:get {:responses {200 {:body {:color string? :services coll?}}}
:handler status}}]]
:handler status}}]]
{:data {
:muuntaja m/instance
:middleware [swagger/swagger-feature
Expand Down
6 changes: 6 additions & 0 deletions api/test/api/end_to_end_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@
(is (= {"test" []} (generate-text-bulk {:document-plan-name "Untitled plan"
:data-rows {"test" {}}}))))

(deftest ^:integration non-async-text-generation
(is (= ["Building Search Applications is authored by Manu Konchady."]
(:variants (generate-text {:document-plan-name "Authorship"
:data-file-name "books.csv"
:async false})))))

(deftest ^:integration text-generation-with-simple-plan
(is (= ["Building Search Applications is authored by Manu Konchady."]
(generate-text {:document-plan-name "Authorship"
Expand Down
15 changes: 8 additions & 7 deletions api/test/api/test_utils.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,15 @@
(< 10 retry-count) (throw (Exception. "Result was not ready after 10 seconds."))
:else (do (Thread/sleep 1000) (recur (inc retry-count) (request!))))))))

(defn generate-text [{:keys [document-plan-name data-file-name reader-flags]}]
(let [{:keys [status body]} (q "/nlg/" :post {:documentPlanId (load-document-plan document-plan-name)
:readerFlagValues (or reader-flags {"Eng" true})
:dataId (if (some? data-file-name)
(load-data-file data-file-name)
"")})]
(defn generate-text [{:keys [document-plan-name data-file-name reader-flags async]}]
(let [{:keys [status body]} (q "/nlg/" :post (cond-> {:documentPlanId (load-document-plan document-plan-name)
:readerFlagValues (or reader-flags {"Eng" true})
:dataId (if (some? data-file-name)
(load-data-file data-file-name)
"")}
(false? async) (assoc :async false)))]
(when (= 200 status)
(get-result (:resultId body)))))
(if (false? async) body (get-result (:resultId body))))))

(defn generate-text-bulk [{:keys [document-plan-name data-rows reader-flags]}]
(let [{:keys [status body]} (q "/nlg/_bulk/" :post {:documentPlanId (load-document-plan document-plan-name)
Expand Down