Skip to content

Commit

Permalink
Data file deletion via API (#333)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rokasramas
rokasramas committed Jul 19, 2021
1 parent f5529ef commit 54b41ca
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 26 deletions.
6 changes: 4 additions & 2 deletions api/src/api/resource.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@
:updatedAt (ts-now),
:variants ["Text value."]}}}
:nlg-bulk {:post {:application/json {:resultIds (take 10 (repeatedly utils/gen-uuid))}}}
:accelerated-text-data-files {:post {:application/json {:message "Succesfully uploaded file"
:id (utils/gen-uuid)}}}
:accelerated-text-data-files {:post {:application/json {:message "Succesfully uploaded file"
:id (utils/gen-uuid)}}
:delete {:application/json {:message "Succesfully deleted file"
:id (utils/gen-uuid)}}}
:health {:get {:application/json {:health "Ok"}}}
:status {:get {:application/json {:color "green"
:services {"service" true}}}}})
Expand Down
39 changes: 28 additions & 11 deletions api/src/api/server.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,17 +109,34 @@
:handler service/delete-result}
:options {:handler cors-handler
:no-doc true}}]
["/accelerated-text-data-files/" {:parameters {:multipart {:file multipart/bytes-part}}
:post (fn [request]
(let [{params :params auth-info :auth-info} (multipart-handler request)
id (data-files/store! (get params "file") (:group-id auth-info))]
{:status 200
:body {:message "Succesfully uploaded file" :id id}}))
:coercion reitit.coercion.spec/coercion
:summary "Upload a file"
:responses {200 {:body {:message string?
:id string?}
:examples (get-in response-examples [:accelerated-text-data-files :post])}}}]
["/accelerated-text-data-files/"
{:post {:parameters {:multipart {:file multipart/bytes-part}}
:coercion reitit.coercion.spec/coercion
:summary "Upload a file"
:responses {200 {:body {:message string?
:id string?}
:examples (get-in response-examples [:accelerated-text-data-files :post])}}
:handler (fn [request]
(let [{params :params auth-info :auth-info} (multipart-handler request)
id (data-files/store! (get params "file") (:group-id auth-info))]
{:status 200
:body {:message "Succesfully uploaded file" :id id}}))}
:delete {:parameters {:body {:id string?}}
:responses {200 {:body {:message string?
:id string?}
:examples (get-in response-examples [:accelerated-text-data-files :delete])}}
:coercion reitit.coercion.spec/coercion
:summary "Delete data file"
:middleware [muuntaja/format-request-middleware
coercion/coerce-request-middleware
coercion/coerce-response-middleware]
:handler (fn [{{{id :id} :body} :parameters auth-info :auth-info}]
(if (data-files/delete-data-file! id (:group-id auth-info))
{:status 200
:body {:message "Succesfully deleted file" :id id}}
{:status 404}))}
:options {:handler cors-handler
:no-doc true}}]
["/swagger.json" {:get {:no-doc true
:swagger {:info {:title "nlg-api"}}
:handler (swagger/create-swagger-handler)}}]
Expand Down
26 changes: 13 additions & 13 deletions api/src/data/entities/data_files.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,15 @@
(db/read! data-files-db (build-key key group-id)))

(defn delete-data-file! [key group-id]
(log/infof "Deleting data file: `%s`" key)
(db/delete! data-files-db (build-key key group-id)))
(when (read-data-file key group-id)
(log/infof "Deleting data file: `%s`" key)
(db/delete! data-files-db (build-key key group-id))))

(defn store!
"Expected keys are :filename and :content everything else is optional"
[{filename :filename :as data-file} group-id]
(let [key (build-key filename group-id)]
(when (some? (read-data-file filename group-id))
(delete-data-file! filename group-id))
(delete-data-file! filename group-id)
(log/infof "Storing data file: `%s`" filename)
(db/write! data-files-db key
#::data-file{:name filename
Expand All @@ -62,12 +62,16 @@
(user-group/link-file group-id key))
filename)

(defn drop-group-id [key]
(when (some? key)
(last (re-find #"(.+?)#(.+)" key))))

(defn parse-data
([data-file]
(parse-data data-file 0 Integer/MAX_VALUE))
([{::data-file/keys [id name content]} offset limit]
(let [[header & rows] (map #(map str/trim %) (cond-> content (some? content) (csv/read-csv)))]
{:id id
{:id (drop-group-id id)
:filename name
:header (vec header)
:rows (take limit (drop offset rows))
Expand Down Expand Up @@ -97,9 +101,9 @@

(defn fetch-most-relevant [id offset limit group-id]
(let [{:keys [filename header rows total]} (some-> (read-data-file id group-id) (parse-data))
sampled-rows (row-selection/sample rows (:relevant-items-limit conf))
m (row-selection/distance-matrix sampled-rows)
selected-rows (drop offset (row-selection/select-rows m sampled-rows limit))]
sampled-rows (row-selection/sample rows (:relevant-items-limit conf))
m (row-selection/distance-matrix sampled-rows)
selected-rows (drop offset (row-selection/select-rows m sampled-rows limit))]
{:id id
:fileName filename
:fieldNames header
Expand All @@ -122,18 +126,14 @@
(some-> (read-data-file id group-id)
(read-content offset limit)))

(defn swap-data-id [{:keys [fileName] :as data}]
(assoc data :id fileName))

(defn listing
([group-id] (listing group-id 0 Integer/MAX_VALUE 0 Integer/MAX_VALUE))
([group-id offset limit recordOffset recordLimit]
(let [data-files (-> (user-group/get-or-create-group group-id) ::ug/data-files)]
{:dataFiles (->> data-files
(drop offset)
(take limit)
(map #(read-content % recordOffset recordLimit))
(map swap-data-id))
(map #(read-content % recordOffset recordLimit)))
:offset offset
:limit limit
:totalCount (count data-files)})))
Expand Down

0 comments on commit 54b41ca

Please sign in to comment.