Skip to content
Browse files

_limit and _offset parameters for REST HTTP requests

  • Loading branch information...
1 parent add42e0 commit 2b2daf9844add8f59e0935c6620558c97d0b6dff @antoniogarrote committed
View
26 src/plaza/rdf/implementations/common.clj
@@ -288,21 +288,25 @@
optional (:optional acum)]
(if (:optional (meta item))
;; add it to the optional elem
- (.addTriplePattern optional (com.hp.hpl.jena.graph.Triple/create (build-query-atom (nth item 0))
- (build-query-atom (nth item 1))
- (build-query-atom (nth item 2))))
+ (let [optg (com.hp.hpl.jena.sparql.syntax.ElementGroup.)]
+ (.addTriplePattern optg (com.hp.hpl.jena.graph.Triple/create (build-query-atom (nth item 0))
+ (build-query-atom (nth item 1))
+ (build-query-atom (nth item 2))))
+ {:building building
+ :optional (conj optional optg)})
;; Is not an optional triple
- (.addTriplePattern building (com.hp.hpl.jena.graph.Triple/create (build-query-atom (nth item 0))
- (build-query-atom (nth item 1))
- (build-query-atom (nth item 2)))))
- {:building building
- :optional optional}))
+ (do (.addTriplePattern building (com.hp.hpl.jena.graph.Triple/create (build-query-atom (nth item 0))
+ (build-query-atom (nth item 1))
+ (build-query-atom (nth item 2))))
+ {:building building
+ :optional optional}))))
{:building (com.hp.hpl.jena.sparql.syntax.ElementGroup.)
- :optional (com.hp.hpl.jena.sparql.syntax.ElementGroup.)}
+ :optional []}
pattern)
built-pattern (do
(when (not (.isEmpty (:optional built-patterns)))
- (.addElement (:building built-patterns) (com.hp.hpl.jena.sparql.syntax.ElementOptional. (:optional built-patterns))))
+ (doseq [optg (:optional built-patterns)]
+ (.addElement (:building built-patterns) (com.hp.hpl.jena.sparql.syntax.ElementOptional. optg))))
(:building built-patterns))
built-filters (loop [bfs (map (fn [f] (build-filter builder f)) (if (nil? (:filters query)) [] (:filters query)))]
(if (not (empty? bfs))
@@ -328,4 +332,6 @@
(.setDistinct built-query true))
(when (:reduced query)
(.setReduced built-query true))
+ (when (:order-by query)
+ (.addOrderBy built-query (keyword-to-variable (:order-by query)) 1))
built-query))))
View
7 src/plaza/rdf/implementations/jena.clj
@@ -42,11 +42,12 @@
([model query query-string]
; (let [query (if (string? query-or-string) (sparql-to-query query-or-string) query-or-string)
; query-string (if (string? query-or-string) query-or-string (str (build-query *sparql-framework* query)))]
-; (println (str "QUERYING JENA WITH: " query-string))
+; (println (str "QUERYING JENA WITH:\r\n" query-string))
(model-critical-read model
(let [qexec (QueryExecutionFactory/create query-string (to-java model))
- ; (let [qexec (QueryExecutionFactory/create (build-query query) @model)
+ ; (let [qexec (QueryExecutionFactory/create (build-query query) @model)
results (iterator-seq (cond (= (:kind query) :select) (.execSelect qexec)))]
+; _results (println (str "BINDING RESULTS: " results))]
(map #(process-model-query-result model %1) results)))))
(defn- model-query-triples-fn
@@ -55,7 +56,7 @@
(let [query (if (string? query-or-string) (sparql-to-query query-or-string) query-or-string)
query-string (if (string? query-or-string) query-or-string (str (build-query *sparql-framework* query-or-string)))
results (model-query-fn model query query-string)]
- (map #(pattern-bind (:pattern query) %1) results))))
+ (map #(pattern-reject-unbound (pattern-bind (:pattern query) %1)) results))))
;; JENA implementation
View
116 src/plaza/rdf/sparql.clj
@@ -136,6 +136,11 @@
([limit]
(fn [query] (query-set-limit limit))))
+(defn chain-limit
+ "Used to conditionally insert limit in the threaded definition of a query"
+ ([h limit]
+ (if (nil? limit) h (assoc h :limit (if (string? (first limit)) (Integer/parseInt (first limit)) (first limit))))))
+
(defn query-unset-limit
"Removes the limit constrain in the number of results"
([query]
@@ -178,6 +183,11 @@
([offset]
(fn [query] (query-set-offset offset))))
+(defn chain-offset
+ "Used to conditionally insert offset in the threaded definition of a query"
+ ([h offset]
+ (if (nil? offset) h (assoc h :offset (if (string? (first offset)) (Integer/parseInt (first offset)) (first offset))))))
+
(defn query-unset-offset
"Removes the offset constraint in the results"
([query]
@@ -206,6 +216,13 @@
([pattern]
(fn [query] (query-set-pattern query pattern))))
+(defn query-set-order-by
+ "Sets the pattern for the query"
+ ([query order-by]
+ (assoc query :order-by order-by))
+ ([order-by]
+ (fn [query] (query-set-order-by query order-by))))
+
;; Parsing a SPARQL string query into a query pattern representation
(defn- var-to-keyword-fn
@@ -247,6 +264,16 @@
([name & args]
(apply make-filter (cons name args))))
+(defn limit
+ "Defines limit as a filter"
+ ([max]
+ (:expression :limit :args max)))
+
+(defn offset
+ "Defines offset as a filter"
+ ([val]
+ (:expression :offset :args val)))
+
;; Querying a model
(defn model-query
@@ -280,34 +307,53 @@
(triple-object (get binding-map o))
o)]
(if (:optional (meta t))
- (optional [sp pp op])
+ (with-meta [sp pp op] {:optional true})
[sp pp op])))
pattern))))
+(defn pattern-reject-unbound
+ "Binds variables in a pattern rejecting the pattern triples that has not been bound"
+ ([pattern]
+ (filter (fn [[s p o]]
+ (and (not (keyword? s))
+ (not (keyword? p))
+ (not (keyword? o)))) pattern)))
+
+
(defn pattern-apply
"Applies a pattern to a set of triples"
([triples pattern-or-vector & filters-pre]
(let [pattern-pre (if (:pattern (meta pattern-or-vector)) pattern-or-vector (make-pattern pattern-or-vector))
vars-pre (pattern-collect-vars pattern-pre)
vars (if-not (empty? vars-pre) vars-pre [:p])
- [pattern filters] (if-not (empty? vars-pre)
- [pattern-pre filters-pre]
- (let [s (nth (first pattern-pre) 0)
- p (nth (first pattern-pre) 1)
- o (nth (first pattern-pre) 2)]
- [(cons [s ?p o] (rest pattern-pre))
- (cons (f :sameTerm ?p p) filters-pre)]))
+ [pattern filterspp] (if-not (empty? vars-pre)
+ [pattern-pre filters-pre]
+ (let [s (nth (first pattern-pre) 0)
+ p (nth (first pattern-pre) 1)
+ o (nth (first pattern-pre) 2)]
+ [(cons [s ?p o] (rest pattern-pre))
+ (cons (f :sameTerm ?p p) filters-pre)]))
+ [limit filtersp] (if (some #(= :limit (:expression %1)) filterspp)
+ [(:args (first (filter #(= :limit (:expression %1)) filterspp))) (filter #(not (= :limit (:expression %1))) filterspp)]
+ [nil filterspp])
+ [offset filters] (if (some #(= :offset (:expression %1)) filtersp)
+ [(:args (first (filter #(= :offset (:expression %1)) filtersp))) (filter #(not (= :offset (:expression %1))) filtersp)]
+ [nil filtersp])
query (if (empty? filters)
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
- (query-set-distinct))
+ (query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset))
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
(query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset)
(query-set-filters filters)))]
(distinct (model-query-triples (defmodel (model-add-triples triples))
query)))))
@@ -323,24 +369,34 @@
(let [pattern-pre (if (:pattern (meta pattern-or-vector)) pattern-or-vector (make-pattern pattern-or-vector))
vars-pre (pattern-collect-vars pattern-pre)
vars (if-not (empty? vars-pre) vars-pre [:p])
- [pattern filters] (if-not (empty? vars-pre)
- [pattern-pre filters-pre]
- (let [s (nth (first pattern-pre) 0)
- p (nth (first pattern-pre) 1)
- o (nth (first pattern-pre) 2)]
- [(cons [s ?p o] (rest pattern-pre))
- (cons (f :sameTerm ?p p) filters-pre)]))
+ [pattern filterspp] (if-not (empty? vars-pre)
+ [pattern-pre filters-pre]
+ (let [s (nth (first pattern-pre) 0)
+ p (nth (first pattern-pre) 1)
+ o (nth (first pattern-pre) 2)]
+ [(cons [s ?p o] (rest pattern-pre))
+ (cons (f :sameTerm ?p p) filters-pre)]))
+ [limit filtersp] (if (some #(= :limit (:expression %1)) filterspp)
+ [(:args (first (filter #(= :limit (:expression %1)) filterspp))) (filter #(not (= :limit (:expression %1))) filterspp)]
+ [nil filterspp])
+ [offset filters] (if (some #(= :offset (:expression %1)) filtersp)
+ [(:args (first (filter #(= :offset (:expression %1)) filtersp))) (filter #(not (= :offset (:expression %1))) filtersp)]
+ [nil filtersp])
query (if (empty? filters)
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
- (query-set-distinct))
+ (query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset))
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
(query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset)
(query-set-filters filters)))]
(distinct (model-query-triples model
query)))))
@@ -387,25 +443,35 @@
(let [pattern-pre (if (:pattern (meta pattern-or-vector)) pattern-or-vector (make-pattern pattern-or-vector))
vars-pre (pattern-collect-vars pattern-pre)
vars (if-not (empty? vars-pre) vars-pre [:p])
- [pattern filters-pre] (if-not (empty? vars-pre)
- [pattern-pre filters-pre]
- (let [s (nth (first pattern-pre) 0)
- p (nth (first pattern-pre) 1)
- o (nth (first pattern-pre) 2)]
- [(cons [s ?p o] (rest pattern-pre))
- (cons (f :sameTerm ?p p) filters-pre)]))
+ [pattern filterspp] (if-not (empty? vars-pre)
+ [pattern-pre filters-pre]
+ (let [s (nth (first pattern-pre) 0)
+ p (nth (first pattern-pre) 1)
+ o (nth (first pattern-pre) 2)]
+ [(cons [s ?p o] (rest pattern-pre))
+ (cons (f :sameTerm ?p p) filters-pre)]))
+ [limit filtersp] (if (some #(= :limit (:expression %1)) filterspp)
+ [(:args (first (filter #(= :limit (:expression %1)) filterspp))) (filter #(not (= :limit (:expression %1))) filterspp)]
+ [nil filterspp])
+ [offset filters] (if (some #(= :offset (:expression %1)) filtersp)
+ [(:args (first (filter #(= :offset (:expression %1)) filtersp))) (filter #(not (= :offset (:expression %1))) filtersp)]
+ [nil filtersp])
filters (concat filters-pre (make-cw-filters pattern))
query (if (empty? filters)
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
- (query-set-distinct))
+ (query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset))
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
(query-set-distinct)
+ (chain-limit limit)
+ (chain-offset offset)
(query-set-filters filters)))]
(distinct (model-query-triples model
query)))))
View
61 src/plaza/rest/core.clj
@@ -157,10 +157,11 @@
(defn apply-resource-argument-map [params mapping]
(let [ks (keys mapping)]
- (reduce (fn [acum k] (let [{uri :uri f :mapper} (k mapping)
- arg (get params (keyword-to-string k))]
- (if (nil? arg) acum (conj acum [uri (f arg)]))))
- [] ks)))
+ (last
+ (reduce (fn [[c acum] k] (let [{uri :uri f :mapper} (k mapping)
+ arg (get params (keyword-to-string k))]
+ (if (nil? arg) [(inc c) (conj acum [uri (keyword (str "?o" c))])] [c (conj acum [uri (f arg)])])))
+ [0 []] ks))))
(defn random-uuid []
(.replace (str (java.util.UUID/randomUUID)) "-" ""))
@@ -170,16 +171,25 @@
(defn build-triples-from-resource-map [uri mapping]
(let [resource-uri (if (seq? uri) (apply rdf-resource uri) (rdf-resource uri))]
- (map #(cons resource-uri %1) mapping)))
+ (reduce (fn [acum [p o]]
+ (if (keyword? o)
+ acum
+ (conj acum [resource-uri p o])))
+ [] mapping)))
(defn build-query-from-resource-map [mapping resource-type]
- (concat [[?s ?p ?o]
- [?s rdf:type resource-type]]
- (vec (map #(cons ?s %1) mapping))))
+ (concat [[?s rdf:type resource-type]]
+ (vec (map (fn [[p o]]
+ (if (keyword? o)
+ (optional [?s p o])
+ [?s p o])) mapping))))
(defn build-single-resource-query-from-resource-map [mapping resource-id]
- (concat [[resource-id ?p ?o]]
- (vec (map #(cons resource-id %1) mapping))))
+ (concat [[resource-id rdf:type :?rt]]
+ (vec (map (fn [[p o]]
+ (if (keyword? o)
+ (optional [resource-id p o])
+ [resource-id p o])) mapping))))
(defn build-single-resource-all-triples-query [resource-id]
[[resource-id ?p ?o]])
@@ -529,8 +539,22 @@
(dissoc "_callback"))]
(-> request-methods (assoc :jsonp-callback callback)
(assoc :params paramsp)))
- request-methods))]
- jsonp-request)))
+ request-methods))
+ limit-request (let [limit (get (:params jsonp-request) "_limit")]
+ (if-not (nil? limit)
+ (let [paramsp (-> (:params jsonp-request)
+ (dissoc "_limit"))]
+ (-> jsonp-request (assoc :limit limit)
+ (assoc :params paramsp)))
+ jsonp-request))
+ offset-request (let [offset (get (:params limit-request) "_offset")]
+ (if-not (nil? offset)
+ (let [paramsp (-> (:params limit-request)
+ (dissoc "_offset"))]
+ (-> limit-request (assoc :offset offset)
+ (assoc :params paramsp)))
+ limit-request))]
+ offset-request)))
(defn check-tbox-request
"Checks if the request is asking for TBox metadata instead of the actual service data"
@@ -558,7 +582,6 @@
query (build-single-resource-query-from-resource-map mapping id)
results (rd (ts (:resource-ts environment)) query)
triples (distinct (flatten-1 results))]
- (log :info (str "GET REQUEST -> mapping:" mapping " triples:" triples " for query " query " and id " id))
{:body (render-triples triples (mime-to-format request) (:resource environment) request)
:headers {"Content-Type" (format-to-mime request)}
:status 200}))
@@ -570,10 +593,7 @@
triples-to-update (conj triples-pre [id rdf:type (:resource-type environment)])
results (swap (ts (:resource-ts environment)) query triples-to-update)
triples (distinct (flatten-1 results))]
- (log :info (str "PUT REQUEST -> mapping:" mapping " query:" query))
- (log :info (str "QUERY"))
(doseq [t query] (log :info t))
- (log :info (str "VALUES"))
(doseq [t triples-to-update] (log :info t))
{:body (render-triples triples (mime-to-format request) (:resource environment) request)
:headers {"Content-Type" (format-to-mime request)}
@@ -584,7 +604,6 @@
query (build-single-resource-query-from-resource-map mapping id)
results (in (ts (:resource-ts environment)) query)
triples (distinct (flatten-1 results))]
- (log :info (str "DELETE REQUEST -> mapping:" mapping " triples:" triples))
{:body (render-triples triples (mime-to-format request) (:resource environment) request)
:headers {"Content-Type" (format-to-mime request)}
:status 200}))
@@ -593,9 +612,11 @@
(defn handle-get-collection [request environment]
(let [mapping (apply-resource-argument-map (:params request) (:resource-map environment))
query (build-query-from-resource-map mapping (:resource-type environment))
- results (rd (ts (:resource-ts environment)) query)
+ results (let [rd-argspp []
+ rd-argsp (if (:limit request) (conj rd-argspp (f :limit (:limit request))) rd-argspp)
+ rd-args (if (:offset request) (conj rd-argsp (f :offset (:offset request))) rd-argsp)]
+ (rd (ts (:resource-ts environment)) query rd-args))
triples (distinct (flatten-1 results))]
- (log :info (str "GET REQUEST -> mapping:" mapping " triples:" triples))
{:body (render-triples triples (mime-to-format request) (:resource environment) request)
:headers {"Content-Type" (format-to-mime request)}
:status 200}))
@@ -605,7 +626,6 @@
resource-id ((:id-gen-function environment) request environment)
triples-pre (build-triples-from-resource-map resource-id mapping)
triples (conj triples-pre [resource-id rdf:type (:resource-type environment)])]
- (log :info (str "POST REQUEST -> id:" resource-id " mapping:" mapping " triples:" triples))
(out (ts (:resource-ts environment)) triples)
{:body (render-triples triples :xml (:resource environment) request)
:headers {"Content-Type" "application/xml"}
@@ -616,7 +636,6 @@
query (build-query-from-resource-map mapping (:resource-type environment))
results (in (ts (:resource-ts environment)) query)
triples (distinct (flatten-1 results))]
- (log :info (str "DELETE REQUEST -> mapping:" mapping " query:" query))
{:body (render-triples triples (mime-to-format request) (:resource environment) request)
:headers {"Content-Type" (format-to-mime request)}
:status 200}))
View
30 src/plaza/triple_spaces/distributed_server.clj
@@ -22,22 +22,32 @@
(let [pattern-pre (if (:pattern (meta pattern-or-vector)) pattern-or-vector (make-pattern pattern-or-vector))
vars-pre (pattern-collect-vars pattern-pre)
vars (if-not (empty? vars-pre) vars-pre [:p])
- [pattern filters] (if-not (empty? vars-pre)
- [pattern-pre filters-pre]
- (let [s (nth (first pattern-pre) 0)
- p (nth (first pattern-pre) 1)
- o (nth (first pattern-pre) 2)]
- [(cons [s ?p o] (rest pattern-pre))
- (cons (f :sameTerm ?p p) filters-pre)]))
+ [pattern filterspp] (if-not (empty? vars-pre)
+ [pattern-pre filters-pre]
+ (let [s (nth (first pattern-pre) 0)
+ p (nth (first pattern-pre) 1)
+ o (nth (first pattern-pre) 2)]
+ [(cons [s ?p o] (rest pattern-pre))
+ (cons (f :sameTerm ?p p) filters-pre)]))
+ [limit filtersp] (if (some #(= :limit (:expression %1)) filterspp)
+ [(:args (first (filter #(= :limit (:expression %1)) filterspp))) (filter #(not (= :limit (:expression %1))) filterspp)]
+ [nil filterspp])
+ [offset filters] (if (some #(= :offset (:expression %1)) filtersp)
+ [(:args (first (filter #(= :offset (:expression %1)) filtersp))) (filter #(not (= :offset (:expression %1))) filtersp)]
+ [nil filtersp])
query (if (empty? filters)
(defquery
(query-set-pattern pattern)
(query-set-type :select)
- (query-set-vars vars))
+ (query-set-vars vars)
+ (chain-limit limit)
+ (chain-offset offset))
(defquery
(query-set-pattern pattern)
(query-set-type :select)
(query-set-vars vars)
+ (chain-limit limit)
+ (chain-offset offset)
(query-set-filters filters)))]
query)))
@@ -108,7 +118,6 @@
ts (query-triples model query)]
(if (empty? ts)
(let [prom (promise)]
- (log :info "storing remote state in queue for rdb")
(plaza.triple-spaces.multi-remote-server.auxiliary/store-blocking-rd queue [(query-to-string query) :rdb (:client-id options)])
(process-in-blocking-op name false rabbit-conn options prom)
@prom) ts)))
@@ -122,9 +131,6 @@
(let [triples-to-remove (query-triples model (gen-query pattern filters))]
(if (empty? triples-to-remove) triples-to-remove
(let [flattened-triples (flatten-1 triples-to-remove)]
- (log :error (str "removing triplesb!!!!!"))
- (log :error (str "flattened: " flattened-triples))
- (doseq [t flattened-triples] (log :error (str "TRIPLE: " t)))
;; deleting read triples
(with-model model (model-remove-triples flattened-triples))
;; delivering notifications
View
8 test/plaza/rdf/sparql_test.clj
@@ -1,4 +1,5 @@
(ns plaza.rdf.sparql-test
+ (:use [plaza.utils])
(:use [plaza.rdf predicates core sparql] :reload-all)
(:use [plaza.rdf.implementations jena] :reload-all)
(:use [clojure.test]))
@@ -182,3 +183,10 @@
(is (not (= (.indexOf (to-string s) "ba") -1)))
(is (not (= (.indexOf (to-string p) "type") -1)))
(is (not (= (.indexOf (to-string o) "Post") -1)))))
+
+(deftest test-model-pattern-apply-limit-offset
+ (let [m (defmodel (model-add-triples [[:a :b :c] [:a :b :e] [:a :b :f]]))
+ result (flatten-1 (model-pattern-apply m [[:a :b ?p]] (f :limit 2) (f :offset 0)))
+ result2 (flatten-1 (model-pattern-apply m [[:a :b ?p]] (f :limit 2) (f :offset 2)))]
+ (is (= 2 (count result)))
+ (is (= 1 (count result2)))))
View
22 test/plaza/rest/core_test.clj
@@ -28,7 +28,7 @@
(defn- build-mulgara
([] (build-model :mulgara :rmi "rmi://localhost/server1")))
-(defonce *should-test* true)
+(defonce *should-test* false)
(when *should-test*
@@ -149,7 +149,6 @@
(let [subj (str (first (first (model-to-triples m))))
res2 (clojure-http.resourcefully/get (str subj ".js3"))
ts (clojure.contrib.json/read-json (apply str (:body-seq res2)))]
- (doseq [t ts] (log :info t))
(is (= 3 (count ts))))))
(deftest test-del-post-get-json
@@ -213,6 +212,25 @@
(is (= (str (first (first (model-to-triples m2))))
(str (first (first (model-to-triples m)))))))))
+ (deftest test-del-get-offset-limit
+ (println "***************************************************\n GET OFFSET LIMIT \n******************************************************")
+ (clojure-http.resourcefully/delete "http://localhost:8082/Agent")
+ (let [res1 (clojure-http.resourcefully/post "http://localhost:8082/Agent?age=20&gender=male")
+ res2 (clojure-http.resourcefully/post "http://localhost:8082/Agent?age=21&gender=male")
+ res3 (clojure-http.resourcefully/post "http://localhost:8082/Agent?age=22&gender=male")
+ m1 (build-model :jena)
+ m2 (build-model :jena)
+ some0 (clojure-http.resourcefully/get "http://localhost:8082/Agent.n3")
+ some1 (clojure-http.resourcefully/get "http://localhost:8082/Agent.n3?_limit=2&_offset=0")
+ some2 (clojure-http.resourcefully/get "http://localhost:8082/Agent.n3?_limit=2&_offset=2")]
+ (log :error (str "*** RES 0\r\n" some0))
+ (log :error (str "*** RES 1\r\n" some1))
+ (log :error (str "*** RES 2\r\n" some2))
+ (with-model m1 (document-to-model (java.io.ByteArrayInputStream. (.getBytes (apply str (:body-seq some1)))) :n3))
+ (with-model m2 (document-to-model (java.io.ByteArrayInputStream. (.getBytes (apply str (:body-seq some2)))) :n3))
+ (is (= 6 (count (model-to-triples m1))))
+ (is (= 3 (count (model-to-triples m2))))))
+
(catch Exception ex (throw ex))
(finally (do (log :info "CLEANING...")

0 comments on commit 2b2daf9

Please sign in to comment.
Something went wrong with that request. Please try again.