Skip to content

Commit

Permalink
[MLv2] Add preview-expression to eval an expression on sample data
Browse files Browse the repository at this point in the history
This is needed to show the previews in the UX for the combining columns
epic #39977.

Fixes #39979.
  • Loading branch information
bshepherdson committed Apr 16, 2024
1 parent f21656d commit 153cb3d
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 1 deletion.
4 changes: 4 additions & 0 deletions src/metabase/lib/core.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
[metabase.lib.drill-thru.pivot :as lib.drill-thru.pivot]
[metabase.lib.equality :as lib.equality]
[metabase.lib.expression :as lib.expression]
[metabase.lib.expression.preview :as lib.expression.preview]
[metabase.lib.fe-util :as lib.fe-util]
[metabase.lib.field :as lib.field]
[metabase.lib.filter :as lib.filter]
Expand Down Expand Up @@ -51,6 +52,7 @@
lib.drill-thru.pivot/keep-me
lib.equality/keep-me
lib.expression/keep-me
lib.expression.preview/keep-me
lib.field/keep-me
lib.filter/keep-me
lib.filter.update/keep-me
Expand Down Expand Up @@ -172,6 +174,8 @@
rtrim
upper
lower]
[lib.expression.preview
preview-expression]
[lib.fe-util
dependent-metadata
expression-clause
Expand Down
80 changes: 80 additions & 0 deletions src/metabase/lib/expression/preview.cljc
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
(ns metabase.lib.expression.preview
"**Previewing** an expression means evaluating it on sample rows of data.
This is used in the FE to give example results of expressions under construction, eg. when combining columns with
`:concat`."
(:require
[clojure.string :as str]
[metabase.lib.dispatch :as lib.dispatch]
[metabase.lib.equality :as lib.equality]
[metabase.lib.hierarchy :as lib.hierarchy]))

(defmulti preview-expression
"Given a `query` and `stage-number`, an `expression` (a pMBQL expression clause or a literal like a string or number)
a list `columns` and a `col->value` function, returns the value of `expression` applied to that data.
It is **legal** for the expression to refer to columns that aren't found in the `columns`! They might be a new
implicit join, or otherwise not found in the columns. In that case, [[preview-expression]] throws an exception.
**NOTE:** This does not support all expressions at present! Many of our expressions are quite complex to evaluate
outside of a database engine. This function only supports a subset of expression functions."
{:arglists '([query stage-number expression columns col->value])}
(fn [_query _stage-number expression _columns _col->value]
(lib.dispatch/dispatch-value expression))
:hierarchy lib.hierarchy/hierarchy)

;; Default: Throw for an unrecognized expression.
(defmethod preview-expression :default
[query stage-number expression columns _col->value]
(throw (ex-info (str "Unable to preview expression: " (pr-str expression))
{:query query
:stage-number stage-number
:expression expression
:columns columns})))

;; For literals (ie. anything which is neither an MBQL clause nor a `:lib/type` MLv2 structure), preview-expression
;; returns them as they are.
(defmethod preview-expression :dispatch-type/*
[_query _stage-number expression _columns _col->value]
expression)

;; Refs get resolved to their corresponding value in the row.
(defmethod preview-expression :field
[query stage-number field-ref columns col->value]
(let [column (lib.equality/find-matching-column field-ref columns)
value (if column
(col->value column ::not-found) ; `nil` might be a legit NULL!
::not-found)]
(if (not= value ::not-found)
value
(throw (ex-info "Failed to find field in preview-expression" {:query query
:stage-number stage-number
:field-ref field-ref
:columns columns
:col->value col->value})))))

;; Arithmetic
(defmethod preview-expression :+
[query stage-number [_+ _opts & args] columns col->value]
(reduce + 0 (for [arg args]
(preview-expression query stage-number arg columns col->value))))

(defmethod preview-expression :-
[query stage-number [_- _opts a b] columns col->value]
(- (preview-expression query stage-number a columns col->value)
(preview-expression query stage-number b columns col->value)))

(defmethod preview-expression :*
[query stage-number [_* _opts a b] columns col->value]
(* (preview-expression query stage-number a columns col->value)
(preview-expression query stage-number b columns col->value)))

(defmethod preview-expression :/
[query stage-number [_slash _opts a b] columns col->value]
(double (/ (preview-expression query stage-number a columns col->value)
(preview-expression query stage-number b columns col->value))))

(defmethod preview-expression :concat
[query stage-number [_concat _opts & strings] columns col->value]
(str/join (for [s strings]
(str (preview-expression query stage-number s columns col->value)))))
14 changes: 13 additions & 1 deletion src/metabase/lib/hierarchy.cljc
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
(ns metabase.lib.hierarchy
(:refer-clojure :exclude [derive isa?]))
(:refer-clojure :exclude [derive isa?])
(:require
[metabase.util :as u]))

;; metabase.util is needed for the side effect of registering the `:dispatch-type/*` hierarchy.
(comment u/keep-me)

(defonce ^{:doc "Keyword hierarchy for MLv2 stuff."} hierarchy
(atom (make-hierarchy)))
Expand All @@ -11,6 +16,13 @@
;; for REPL convenience so we don't dump a lot of garbage
nil)

;; Find all the descendants of `:dispatch-type/*` in [[metabase.util]] and duplicate those relationships in the new
;; hierarchy.
(doseq [dtype (descendants :dispatch-type/*)
parent (parents dtype)]
#_(prn "derive" dtype parent)
(derive dtype parent))

(defn isa?
"Like [[clojure.core/isa?]], but uses [[hierarchy]]."
[tag parent]
Expand Down
12 changes: 12 additions & 0 deletions src/metabase/lib/js.cljs
Original file line number Diff line number Diff line change
Expand Up @@ -2048,6 +2048,18 @@
expression-position)
clj->js))))

(defn ^:export preview-expression
"Evaluates `an-expression-clause` against a `row` of data as returned from a query. Both `row` and `columns` should be
in the same order, ie. `row[i]` should be the value for `columns[i]`.
Not all expression operators can be evaluated in memory outside the data warehouse. In addition, many operators that
*could* be supported have not been implemented yet - ask if you need something added.
Throws if `an-expression-clause` contains any subexpressions that can't be evaluated in memory."
[a-query stage-number an-expression-clause columns row]
(let [column-values (zipmap columns row)]
(lib.core/preview-expression a-query stage-number an-expression-clause columns column-values)))

;; TODO: [[field-values-search-info]] seems over-specific - I feel like we can do a better job of extracting search info
;; from arbitrary entities, akin to [[display-info]].

Expand Down
87 changes: 87 additions & 0 deletions test/metabase/lib/expression/preview_test.cljc
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
(ns metabase.lib.expression.preview-test
(:require
#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))
[clojure.test :refer [are deftest is testing]]
[medley.core :as m]
[metabase.lib.core :as lib]
[metabase.lib.expression.preview :as lib.expression.preview]
[metabase.lib.test-metadata :as meta]))

(comment lib/keep-me)

#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))

(def ^:private orders-row-values
{"ID" "3"
"USER_ID" "1"
"PRODUCT_ID" "105"
"SUBTOTAL" 52.723521442619514
"TAX" 2.9
"TOTAL" 49.206842233769756
"DISCOUNT" nil
"CREATED_AT" "2025-12-06T22:22:48.544+02:00"
"QUANTITY" 2})

(defn- orders-query []
(lib/query meta/metadata-provider (meta/table-metadata :orders)))

(defn- orders-row []
(vec (for [col (lib/returned-columns (orders-query))]
{:column col
:value (get orders-row-values (:name col))})))

(defn- preview [expression]
(lib.expression.preview/preview-expression (orders-query) -1 expression (orders-row)))

(deftest ^:parallel preview-expression-test-1-literals-identity
(testing "literals are returned as-is"
(are [expected expression] (= expected (preview expression))
0 0
1 1
"foo" "foo"
true true
false false
nil nil)))

(deftest ^:parallel preview-expression-test-2-arithmetic-on-literals
(testing "arithmetic works on literal numbers"
(are [expected expression] (= expected (preview expression))
12 [:+ {} 7 5]
2.5 [:/ {} 5 2]
71 [:- {} [:* {} 9 8] 1])))

(deftest ^:parallel preview-expression-test-3-concat-on-literals
(testing "concat works on literal strings"
(are [expected expression] (= expected (preview expression))
"some text goes here" [:concat {} "some text" " " "goes" " " "here"]
"words" [:concat {} "words"])))

(deftest ^:parallel preview-expression-test-4a-field-refs-by-id
(testing "field refs by ID work"
(let [by-name (->> (orders-query)
lib/returned-columns
(m/index-by :name))]
(doseq [[column-name expected] orders-row-values
:let [column (get by-name column-name)
col-ref (lib/ref column)]]
(is (int? (last col-ref))) ; Confirm these are refs by ID.
(is (= expected (preview col-ref)))))))

(deftest ^:parallel preview-expression-test-4b-field-refs-by-name
(testing "field refs by name work"
(let [by-name (->> (orders-query)
lib/returned-columns
(m/index-by :name))]
(doseq [[column-name expected] orders-row-values
:let [column (get by-name column-name)
col-ref (lib/ref (dissoc column :id))]]
(is (string? (last col-ref))) ; Confirm these are refs by name.
(is (= expected (preview col-ref)))))))

(deftest ^:parallel preview-expression-test-5-combined
(testing "everything mixed together"
(is (= (/ (orders-row-values "SUBTOTAL")
(orders-row-values "QUANTITY"))
(preview [:/ {}
[:field {:lib/uuid (str (random-uuid))} (meta/id :orders :subtotal)]
[:field {:lib/uuid (str (random-uuid))} (meta/id :orders :quantity)]])))))

0 comments on commit 153cb3d

Please sign in to comment.