Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history 
commit d46b712
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Mon Jun 25 16:42:33 2018 +0200

    WIP 6

commit 9f939c0
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Fri Jun 22 13:20:54 2018 +0200

    WIP 5 [skip ci]

commit f45b979
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Thu Jun 21 10:03:32 2018 +0200

    WIP 4 [skip ci]

commit a80a51a
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 18:25:58 2018 +0200

    WIP 3

commit f615ec5
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 18:12:57 2018 +0200

    WIP 2

commit 5f590c3
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 14:06:13 2018 +0200

    WIP

commit 8f21c01
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 10:35:02 2018 +0200

    Add transformation specs (using s/multi-spec)

commit 55512f8
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 10:21:17 2018 +0200

    Ignore clojure.spec.alpha/keys in eastwood linting [skip ci]

    Relates to "False positive on clojure spec?"
    jonase/eastwood#227

    https://github.com/jonase/eastwood/blob/2235d8b4d5da8ccc9ec47a96257eee05b9f25c00/resource/eastwood/config/clojure.clj

commit 8e10807
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 20 10:13:16 2018 +0200

    Keywordizing maps so could be spec(ified) later

    Also remove indirection visualisation -> visualisation-impl

commit dc7844a
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Tue Jun 19 10:49:13 2018 +0200

    Add aggregation specs and update related tests

commit 98bfe34
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Tue Jun 19 10:46:51 2018 +0200

    Keywordize aggregation payloads to be spec(ified)

commit 4b7f0cd
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Wed Jun 27 01:17:56 2018 +0200

    Unify a.l.specs.transformations ns aliasing

commit 08c093a
Author: Juan A. Ruz <juanantonioruz@gmail.com>
Date:   Tue Jun 26 13:41:25 2018 +0200

    Add aggregation specs

    using s/multi-spec https://clojure.org/guides/spec#_multi_spec

    Use keywords in aggregation internal impl ... so we can spec them

    Add a.l.specs/dataset and update a.l.specs.transformations accordingly

    Add a.l.aggregation.utils specs

    Add a.l.aggregation.filter specs

    Unify spec namespace patterns

    a.l.l.aggregation.pivot specs

    Use potemkin#import-vars to define all specs import in one place

    Remove unused function load-columns

    Adding specs to bar, line, pie and scatter aggregation types
  • Loading branch information
@tangrammer
tangrammer committed Jun 26, 2018
1 parent ef87c8b commit c2c41e2
Show file tree
Hide file tree
Showing 12 changed files with 317 additions and 128 deletions.
35 changes: 13 additions & 22 deletions backend/src/akvo/lumen/lib/aggregation/bar.clj
View file
Expand Up @@ -12,27 +12,18 @@

(defn query
[tenant-conn {:keys [columns table-name]} query]
(let [filter-sql (filter/sql-str columns (get query "filters"))
(let [filter-sql (filter/sql-str columns (:filters query))
max-elements 200
column-x (utils/find-column columns (get query "bucketColumn"))
column-x-type (get column-x "type")
column-x-name (get column-x "columnName")
column-x-title (get column-x "title")
column-y (utils/find-column columns (get query "metricColumnY"))
column-y-name (get column-y "columnName")
column-y-title (get column-y "title")

column-subbucket (utils/find-column columns (get query "subBucketColumn"))
column-subbucket-name (get column-subbucket "columnName")
column-subbucket-title (get column-subbucket "title")

aggregation-method (get query "metricAggregation")
truncate-size (or (get query "truncateSize") "ALL")
sql-sort-subquery (case (get query "sort")
column-x (utils/find-column columns (:bucketColumn query))
column-y (utils/find-column columns (:metricColumnY query))
column-subbucket (utils/find-column columns (:subBucketColumn query))
aggregation-method (:metricAggregation query)
truncate-size (or (:truncateSize query) "ALL")
sql-sort-subquery (case (:sort query)
nil "ORDER BY x ASC"
"asc" "ORDER BY z.y ASC NULLS FIRST"
"dsc" "ORDER BY z.y DESC NULLS LAST")
sql-sort-subbucket-subquery (case (get query "sort")
sql-sort-subbucket-subquery (case (:sort query)
nil "ORDER BY x ASC"
"asc" "ORDER BY sort_value ASC NULLS FIRST"
"dsc" "ORDER BY sort_value DESC NULLS LAST")
Expand Down Expand Up @@ -78,7 +69,7 @@

valid-spec (boolean (and column-x column-y))
sql-text (if valid-spec (if column-subbucket sql-text-with-subbucket sql-text-without-subbucket) "SELECT NULL")
sql-response (run-query tenant-conn table-name sql-text column-x-name column-y-name filter-sql aggregation-method truncate-size column-subbucket-name)
sql-response (run-query tenant-conn table-name sql-text (:columnName column-x) (:columnName column-y) filter-sql aggregation-method truncate-size (:columnName column-subbucket))
bucket-values (distinct
(mapv
(fn [[x-value y-value s-value]] x-value)
Expand All @@ -96,12 +87,12 @@

(lib/ok
(if (not column-subbucket)
{"series" [{"key" column-y-title
"label" column-y-title
{"series" [{"key" (:title column-y)
"label" (:title column-y)
"data" (mapv (fn [[x-value y-value]]
{"value" y-value})
sql-response)}]
"common" {"metadata" {"type" column-x-type}
"common" {"metadata" {"type" (:type column-x)}
"data" (mapv (fn [[x-value y-value]]
{"label" x-value "key" x-value})
sql-response)}}
Expand All @@ -127,7 +118,7 @@

"common"
{"metadata"
{"type" column-x-type}
{"type" (:type column-x)}
"data" (mapv
(fn [bucket] {"label" bucket "key" bucket})
bucket-values)}})))))
26 changes: 10 additions & 16 deletions backend/src/akvo/lumen/lib/aggregation/line.clj
View file
Expand Up @@ -12,18 +12,12 @@

(defn query
[tenant-conn {:keys [columns table-name]} query]
(let [filter-sql (filter/sql-str columns (get query "filters"))
column-x (utils/find-column columns (get query "metricColumnX"))
column-x-type (get column-x "type")
column-x-name (get column-x "columnName")
column-x-title (get column-x "title")
column-y (utils/find-column columns (get query "metricColumnY"))
column-y-type (get column-y "type")
column-y-name (get column-y "columnName")
column-y-title (get column-y "title")
(let [filter-sql (filter/sql-str columns (:filters query))
column-x (utils/find-column columns (:metricColumnX query))
column-y (utils/find-column columns (:metricColumnY query))
max-points 2500
aggregation-method (get query "metricAggregation")
aggregation-method (if (= column-y-type "text") "count" aggregation-method)
aggregation-method (:metricAggregation query)
aggregation-method (if (= (:type column-y) "text") "count" aggregation-method)
sql-aggregation-subquery (case aggregation-method
nil ""
("min" "max" "count" "sum") (str aggregation-method "(%2$s)")
Expand All @@ -34,19 +28,19 @@
"q3" "percentile_cont(0.75) WITHIN GROUP (ORDER BY %2$s)")
sql-text-with-aggregation (str "SELECT * FROM (SELECT * FROM (SELECT %1$s, " sql-aggregation-subquery " FROM %3$s WHERE %4$s GROUP BY %1$s)z ORDER BY random() LIMIT %6$s)zz ORDER BY zz.%1$s")
sql-text-without-aggregation "SELECT * FROM (SELECT * FROM (SELECT %1$s AS x, %2$s AS y FROM %3$s WHERE %4$s)z ORDER BY random() LIMIT %6$s)zz ORDER BY zz.x"
sql-text-no-x (str "SELECT row_number() over() AS x, " (if (= column-y-type "text") "COUNT(%2$s) AS y " "%2$s AS y ") "FROM %3$s WHERE %4$s GROUP BY %2$s")
sql-text-no-x (str "SELECT row_number() over() AS x, " (if (= (:type column-y) "text") "COUNT(%2$s) AS y " "%2$s AS y ") "FROM %3$s WHERE %4$s GROUP BY %2$s")
sql-text (cond
(not column-x) sql-text-no-x
aggregation-method sql-text-with-aggregation
:else sql-text-without-aggregation)
sql-response (run-query tenant-conn table-name sql-text column-x-name column-y-name filter-sql aggregation-method max-points)]
sql-response (run-query tenant-conn table-name sql-text (:columnName column-x) (:columnName column-y) filter-sql aggregation-method max-points)]
(lib/ok
{"series" [{"key" column-y-title
"label" column-y-title
{"series" [{"key" (:title column-y)
"label" (:title column-y)
"data" (mapv (fn [[x-value y-value]]
{"value" y-value})
sql-response)}]
"common" {"metadata" {"type" column-x-type "sampled" (= (count sql-response) max-points)}
"common" {"metadata" {"type" (:type column-x) "sampled" (= (count sql-response) max-points)}
"data" (mapv (fn [[x-value y-value]]
{"timestamp" x-value})
sql-response)}})))
5 changes: 2 additions & 3 deletions backend/src/akvo/lumen/lib/aggregation/pie.clj
View file
Expand Up @@ -15,7 +15,6 @@
[tenant-conn {:keys [columns table-name]} query]
(let [filter-sql (filter/sql-str columns (:filters query))
bucket-column (utils/find-column columns (:bucketColumn query))
bucket-column-title (:title bucket-column)
counts (run-query tenant-conn table-name (:columnName bucket-column) filter-sql)
max-segments 50]
(if (> (count counts) max-segments)
Expand All @@ -25,8 +24,8 @@
"max" max-segments
"count" (count counts)})
(lib/ok
{"series" [{"key" bucket-column-title
"label" bucket-column-title
{"series" [{"key" (:title bucket-column)
"label" (:title bucket-column)
"data" (mapv
(fn [[bucket-value bucket-count]]
{"value" bucket-count})
Expand Down
46 changes: 17 additions & 29 deletions backend/src/akvo/lumen/lib/aggregation/scatter.clj
View file
Expand Up @@ -28,49 +28,37 @@

(defn query
[tenant-conn {:keys [columns table-name]} query]
(let [filter-sql (filter/sql-str columns (get query "filters"))
column-x (utils/find-column columns (get query "metricColumnX"))
column-x-type (get column-x "type")
column-x-name (get column-x "columnName")
column-x-title (get column-x "title")
column-y (utils/find-column columns (get query "metricColumnY"))
column-y-type (get column-y "type")
column-y-name (get column-y "columnName")
column-y-title (get column-y "title")
column-label (utils/find-column columns (get query "datapointLabelColumn"))
column-label-type (get column-label "type")
column-label-name (get column-label "columnName")
column-label-title (get column-label "title")
column-bucket (utils/find-column columns (get query "bucketColumn"))
column-bucket-name (get column-bucket "columnName")
column-bucket-title (get column-bucket "title")
(let [filter-sql (filter/sql-str columns (:filters query))
column-x (utils/find-column columns (:metricColumnX query))
column-y (utils/find-column columns (:metricColumnY query))
column-label (utils/find-column columns (:datapointLabelColumn query))
column-bucket (utils/find-column columns (:bucketColumn query))
max-points 2500
have-aggregation (boolean column-bucket)
aggregation-method (get query "metricAggregation")
aggregation-method (:metricAggregation query)

sql-text-with-aggregation (str "SELECT "
(sql-aggregation-subquery aggregation-method "%1$s" column-x-type)
(sql-aggregation-subquery aggregation-method "%1$s" (:type column-x))
" AS x, "
(sql-aggregation-subquery aggregation-method "%2$s" column-y-type)
(sql-aggregation-subquery aggregation-method "%2$s" (:type column-y))
" AS y, %8$s AS label FROM (SELECT * FROM %3$s WHERE %4$s ORDER BY random() LIMIT %6$s)z GROUP BY %8$s")
sql-text-without-aggregation "
SELECT * FROM (SELECT * FROM (SELECT %1$s AS x, %2$s AS y, %7$s AS label FROM %3$s WHERE %4$s)z ORDER BY random() LIMIT %6$s)zz ORDER BY zz.x"
sql-text (if have-aggregation sql-text-with-aggregation sql-text-without-aggregation)
sql-response (run-query tenant-conn table-name sql-text column-x-name column-y-name filter-sql aggregation-method max-points column-label-name column-bucket-name)]
sql-text (if (boolean column-bucket) sql-text-with-aggregation sql-text-without-aggregation)
sql-response (run-query tenant-conn table-name sql-text (:columnName column-x) (:columnName column-y) filter-sql aggregation-method max-points (:columnName column-label) (:columnName column-bucket))]
(lib/ok
{"series" [{"key" column-x-title
"label" column-x-title
{"series" [{"key" (:title column-x)
"label" (:title column-x)
"data" (mapv (fn [[x-value y-value label]]
{"value" x-value})
sql-response)
"metadata" {"type" column-x-type}}
{"key" column-y-title
"label" column-y-title
"metadata" {"type" (:type column-x)}}
{"key" (:title column-y)
"label" (:title column-y)
"data" (mapv (fn [[x-value y-value label]]
{"value" y-value})
sql-response)
"metadata" {"type" column-y-type}}]
"common" {"metadata" {"type" column-label-type "sampled" (= (count sql-response) max-points)}
"metadata" {"type" (:type column-y)}}]
"common" {"metadata" {"type" (:type column-label) "sampled" (= (count sql-response) max-points)}
"data" (mapv (fn [[x-value y-value label]]
{"label" label})
sql-response)}})))
2 changes: 1 addition & 1 deletion backend/src/akvo/lumen/lib/aggregation/utils.clj
View file
Expand Up @@ -7,4 +7,4 @@
(throw (ex-info "No such column" {:columnName column-name})))))

(defn load-columns [data columns]
(map #(assoc % :column (find-column columns (:column %))) data))
(map #(assoc % :column (find-column columns (:columnName %))) data))

0 comments on commit c2c41e2

Please sign in to comment.