From 38b1a66124b61977cbe6aef6a69bac8ac6c436d7 Mon Sep 17 00:00:00 2001 From: Zach Daniel Date: Thu, 1 May 2025 15:22:31 -0400 Subject: [PATCH] improvement: combination queries See the guide for more. AshSql & AshPostgres parts will come later. --- README.md | 3 +- .../topics/advanced/combination-queries.md | 228 ++++++++++++ lib/ash/actions/read/calculations.ex | 3 + lib/ash/actions/read/read.ex | 332 ++++++++++++------ lib/ash/actions/sort.ex | 159 ++++++--- lib/ash/data_layer/data_layer.ex | 39 ++ lib/ash/data_layer/ets/ets.ex | 321 +++++++++++++++-- lib/ash/expr/expr.ex | 47 +++ lib/ash/filter/filter.ex | 74 ++++ lib/ash/filter/runtime.ex | 41 ++- lib/ash/policy/authorizer/authorizer.ex | 29 +- lib/ash/query/combination.ex | 113 ++++++ lib/ash/query/combination_attr.ex | 4 + lib/ash/query/query.ex | 243 +++++++++++-- lib/ash/query/ref.ex | 17 +- lib/ash/resource.ex | 40 +-- lib/ash/sort/sort.ex | 30 +- mix.exs | 4 +- test/query_test.exs | 240 +++++++++++++ test/type/union_test.exs | 2 +- 20 files changed, 1707 insertions(+), 262 deletions(-) create mode 100644 documentation/topics/advanced/combination-queries.md create mode 100644 lib/ash/query/combination.ex create mode 100644 lib/ash/query/combination_attr.ex diff --git a/README.md b/README.md index 6ea09637f..6b7eed447 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The [Get Started Livebook](documentation/tutorials/get-started.md) **Tutorial** --- -[**Reference**](#reference) documentation is **information-oriented**, covering every Ash module, function, expression, and DSL. It is produced automatically from our source code. Use the sidebar and the top search +[**Reference**](#reference) documentation is **information-oriented**, covering every Ash module, function, expression, and DSL. It is produced automatically from our source code. Use the sidebar and the top search bar to find relevant reference information. Place the text `dsl` before your search to quickly jump to a particular DSL — e.g. try comparing the results of searching for `name` with the results for `dsl name`. --- @@ -90,6 +90,7 @@ bar to find relevant reference information. Place the text `dsl` before your sea - [Monitoring](documentation/topics/advanced/monitoring.md) - [Multitenancy](documentation/topics/advanced/multitenancy.md) - [Reactor](documentation/topics/advanced/reactor.md) +- [Combination Queries](documentation/topics/advanced/combination-queries.md) - [Timeouts](documentation/topics/advanced/timeouts.md) - [Writing Extensions](documentation/topics/advanced/writing-extensions.md) diff --git a/documentation/topics/advanced/combination-queries.md b/documentation/topics/advanced/combination-queries.md new file mode 100644 index 000000000..4519b5cde --- /dev/null +++ b/documentation/topics/advanced/combination-queries.md @@ -0,0 +1,228 @@ +# Combination Queries + +Ash Framework provides a powerful feature called "combination queries" that allows you to combine multiple queries into a single result set, giving you the ability to create complex data retrieval patterns with minimal effort. For SQL data-layers, this feature is implemented using SQL's UNION, INTERSECT, and EXCEPT operations. + +## Overview + +Combination queries let you: + +- Combine multiple distinct queries into a single result set +- Apply different filters, sorting, limits, and calculations to each subquery +- Use operations like union, intersection, and exclusion to define how results should be combined +- Create complex composite queries that would otherwise require multiple separate database calls + +## Syntax + +To use combination queries, you'll work with the following functions: + +```elixir +Ash.Query.combination_of(query, combinations) +``` + +Where `combinations` is a list of combination specifications starting with a base query, followed by additional operations: + +- `Ash.Query.Combination.base/1`: The starting point for your combined query +- `Ash.Query.Combination.union/1`: Combine with the previous results, removing duplicates +- `Ash.Query.Combination.union_all/1`: Combine with the previous results, keeping duplicates +- `Ash.Query.Combination.intersect/1`: Keep only records that appear in both the previous results and this query +- `Ash.Query.Combination.except/1`: Remove records from the previous results that appear in this query + +## Basic Example + +Here's a simple example that combines users who meet different criteria: + +```elixir +User +|> Ash.Query.filter(active == true) +|> Ash.Query.combination_of([ + # Must always begin with a base combination + Ash.Query.Combination.base( + filter: expr(not(on_a_losing_streak)), + sort: [score: :desc], + limit: 10 + ), + Ash.Query.Combination.union( + filter: expr(not(on_a_winning_streak)), + sort: [score: :asc], + limit: 10 + ) +]) +|> Ash.read!() +``` + +This query would return: +- The top 10 active users who are not on a losing streak (sorted by score descending) +- Union with the bottom 10 active users who are not on a winning streak (sorted by score ascending) + +## Using Calculations in Combinations + +One of the most powerful features of combination queries is the ability to create calculations that can be referenced across the combinations: + +```elixir +query = "fred" + +User +|> Ash.Query.filter(active == true) +|> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(trigram_similarity(user_name, ^query) >= 0.5), + calculations: %{ + match_score: calc(trigram_similarity(user_name, ^query), type: :float) + }, + sort: [ + {calc(trigram_similarity(user_name, ^query), type: :float), :desc} + ], + limit: 10 + ), + Ash.Query.Combination.union( + filter: expr(trigram_similarity(email, ^query) >= 0.5), + calculations: %{ + match_score: calc(trigram_similarity(email, ^query), type: :float) + }, + sort: [ + {calc(trigram_similarity(email, ^query), type: :float), :desc} + ], + limit: 10 + ) +]) +|> Ash.read!() +``` + +This example searches for users where either their name or email matches "fred" with a similarity score of at least 0.5, and returns the top 10 matches of each type sorted by their match score. + +## Accessing Combination Values + +To access values from combination queries in your main query, use the `combinations/1` function in your expressions: + +```elixir +User +|> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(organization.name == "bar"), + calculations: %{ + domain: calc("bar", type: :string), + full_name: calc(name <> "@bar", type: :string) + } + ), + Ash.Query.Combination.union_all( + filter: expr(organization.name == "baz"), + calculations: %{ + domain: calc("baz", type: :string), + full_name: calc(name <> "@baz", type: :string) + } + ) +]) +|> Ash.Query.calculate(:email_domain, :string, expr(^combinations(:domain))) +|> Ash.Query.calculate(:display_name, :string, expr(^combinations(:full_name))) +|> Ash.read!() +``` + +In this example, the `combinations(:domain)` and `combinations(:full_name)` references allow the outer query to access the calculation values from the inner combinations. + +## Sorting and Distinct Operations + +You can sort and filter the combined results using the calculations from your combinations: + +```elixir +User +|> Ash.Query.combination_of([ + Ash.Query.Combination.base(calculations: %{sort_order: calc(3, type: :integer)}), + Ash.Query.Combination.union_all( + filter: expr(name == "alice"), + calculations: %{sort_order: calc(1, type: :integer)} + ), + Ash.Query.Combination.union_all( + filter: expr(name == "john"), + calculations: %{sort_order: calc(2, type: :integer)} + ) +]) +|> Ash.Query.sort([{calc(^combinations(:sort_order)), :asc}, {:name, :asc}]) +|> Ash.Query.distinct(:name) +|> Ash.read!() +``` + +This will return results in the order: Alice, John, and then all other users, thanks to the custom sort_order calculation. + +## Important Rules and Limitations + +1. **This is an internal power tool**: No public interfaces like `AshJsonApi`/`AshGraphql` will be + updated to allow this sort of query to be built "from the outside". It is designed to be implemented + within an action, "under the hood". + +2. **Base Combination Required**: Your list of combinations must always start with `Ash.Query.Combination.base/1`. + +3. **Field Consistency**: All combinations must produce the same set of fields. This means: + - If one combination has a calculation, all combinations need that calculation + - Select statements should be consistent across combinations + - If a calculation added to a combination has the same name as an attribute, then it will + be used by `combinations(:that_field)`, allowing for combinations to "override" attribute + values. + +4. **Primary Keys**: When adding runtime calculations or loading related data with `Ash.Query.load/2`, all fieldsets must include the primary key of the resource. If this is not the case, the query will fail. + +5. **Type Specification**: When referencing calculation values with `combinations/1`, the calculation must have been added with a specified type on the `base` query at a minimum: + ```elixir + # Correct - type is specified + calc(expression, type: :string) + + # Incorrect - will raise an error when referenced + calc(expression) + ``` + +## Data Layer Support + +Combination queries depend on data layer support. The implementation in this release includes support for ETS data layer, with implementation for SQL and Postgres to be added in future releases. + +## Performance Considerations + +Combination queries can be more efficient than multiple separate queries, especially when: + +- You need to apply complex ordering or pagination to combined datasets +- You want to deduplicate results across multiple selection criteria +- You need to perform operations like intersection or exclusion between sets + +However, be mindful that complex combinations can generate equally complex SQL queries, so monitor performance in production scenarios. + +## Practical Examples + +### Example 1: Search across multiple fields + +```elixir +Post +|> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(ilike(title, ^("%" <> search_term <> "%"))), + calculations: %{match_type: calc("title", type: :string)}, + sort: [published_at: :desc], + limit: 10 + ), + Ash.Query.Combination.union( + filter: expr(ilike(body, ^("%" <> search_term <> "%"))), + calculations: %{match_type: calc("body", type: :string)}, + sort: [published_at: :desc], + limit: 10 + ) +]) +|> Ash.Query.sort([published_at: :desc]) +|> Ash.Query.calculate(:matched_in, :string, expr(^combinations(:match_type))) +|> Ash.read!() +``` + +### Example 2: Complex filtering with intersection + +```elixir +User +|> Ash.Query.combination_of([ + Ash.Query.Combination.base(filter: expr(role == "admin")), + Ash.Query.Combination.intersect(filter: expr(last_login > ^one_month_ago)) +]) +|> Ash.read!() +``` + +This returns all admin users who have logged in within the last month. + +## Summary + +Combination queries provide a powerful tool for creating complex data retrieval patterns in Ash. By combining multiple queries with different filters, sorts, and calculations, you can build sophisticated interfaces that would otherwise require multiple database queries and application-level merging of results. + +This feature is particularly valuable for search interfaces, reporting tools, and anywhere you need to blend data from different filter conditions in a single, cohesive result set. diff --git a/lib/ash/actions/read/calculations.ex b/lib/ash/actions/read/calculations.ex index 57c28fc13..0577d8003 100644 --- a/lib/ash/actions/read/calculations.ex +++ b/lib/ash/actions/read/calculations.ex @@ -1106,6 +1106,9 @@ defmodule Ash.Actions.Read.Calculations do {{:__calc_dep__, _}, _}, acc -> acc + {{:__ash_runtime_sort__, _}, _}, acc -> + acc + {name, calculation}, acc -> Map.put(acc, name, calculation) end) diff --git a/lib/ash/actions/read/read.ex b/lib/ash/actions/read/read.ex index aed66e894..52e9bd2d9 100644 --- a/lib/ash/actions/read/read.ex +++ b/lib/ash/actions/read/read.ex @@ -472,7 +472,7 @@ defmodule Ash.Actions.Read do {:ok, query} <- authorize_query(query, opts), {:ok, sort} <- add_calc_context_to_sort( - query, + query.sort, opts[:actor], opts[:authorize?], query.tenant, @@ -480,6 +480,7 @@ defmodule Ash.Actions.Read do query.resource, query.domain, parent_stack: parent_stack_from_context(query.context), + first_combination: Enum.at(query.combination_of, 0), source_context: query.context ), query <- %{ @@ -608,6 +609,7 @@ defmodule Ash.Actions.Read do opts ), {:ok, query} <- paginate(query, action, opts[:skip_pagination?]), + :ok <- validate_combinations(query, calculations_at_runtime, query.load), {:ok, data_layer_query} <- Ash.Query.data_layer_query(query, data_layer_calculations: data_layer_calculations), {{:ok, results}, query} <- @@ -663,6 +665,53 @@ defmodule Ash.Actions.Read do end end + defp validate_combinations(%{combination_of: []}, _, _) do + :ok + end + + defp validate_combinations(query, runtime_calculations, load) do + case query.combination_of do + [%{type: type} | _] when type != :base -> + {:error, "Invalid combinations. The first combination must have type `:base`."} + + combination_of -> + default_select = + MapSet.to_list(Ash.Resource.Info.selected_by_default_attribute_names(query.resource)) + + fieldsets = + Enum.map( + combination_of, + &Enum.sort(Enum.uniq((&1.select || default_select) ++ Map.keys(&1.calculations))) + ) + + case Enum.uniq(fieldsets) do + [fieldset] -> + if requires_pkey_but_not_selecting_it(query, fieldset, runtime_calculations, load) do + :ok + else + {:error, + "When runtime calculations or loads are present, all fieldsets must contain the primary key of the resource."} + end + + _ -> + {:error, + """ + Invalid combinations. All fieldsets must be the same, got the following: + + #{inspect(query.combination_of)} + """} + end + end + end + + defp requires_pkey_but_not_selecting_it(query, fieldset, runtime_calculations, load) do + (Enum.empty?(runtime_calculations) && Enum.empty?(load)) || + Enum.all?( + Ash.Resource.Info.primary_key(query.resource), + &Enum.member?(fieldset, Atom.to_string(&1)) + ) + end + defp add_relationship_count_aggregates(query) do Enum.reduce(query.load, query, fn {relationship_name, related_query}, query -> relationship = Ash.Resource.Info.relationship(query.resource, relationship_name) @@ -862,6 +911,7 @@ defmodule Ash.Actions.Read do |> Ash.Filter.hydrate_refs(%{ resource: query.resource, public?: false, + first_combination: Enum.at(query.combination_of, 0), parent_stack: parent_stack_from_context(query.context) }) |> case do @@ -1188,113 +1238,114 @@ defmodule Ash.Actions.Read do end end - defp hydrate_sort(%{sort: empty} = query, _actor, _authorize?, _tenant, _tracer, _domain) - when empty in [nil, []] do - {:ok, query} - end - defp hydrate_sort(query, actor, authorize?, tenant, tracer, domain) do - query.sort - |> List.wrap() - |> Enum.map(fn {field, direction} -> - if is_atom(field) do - case Ash.Resource.Info.field(query.resource, field) do - %Ash.Resource.Calculation{} = calc -> {calc, direction} - %Ash.Resource.Aggregate{} = agg -> {agg, direction} - _field -> {field, direction} + [:sort, :distinct, :distinct_sort] + |> Enum.reject(&(Map.get(query, &1) in [nil, []])) + |> Enum.reduce({:ok, query}, fn key, {:ok, query} -> + query + |> Map.get(key) + |> Enum.map(fn {field, direction} -> + if is_atom(field) do + case Ash.Resource.Info.field(query.resource, field) do + %Ash.Resource.Calculation{} = calc -> {calc, direction} + %Ash.Resource.Aggregate{} = agg -> {agg, direction} + _field -> {field, direction} + end + else + {field, direction} end - else - {field, direction} - end - end) - |> Enum.reduce_while({:ok, []}, fn - {%Ash.Resource.Calculation{} = resource_calculation, direction}, {:ok, sort} -> - case Ash.Query.Calculation.from_resource_calculation(query.resource, resource_calculation, - source_context: query.context - ) do - {:ok, calc} -> - case hydrate_calculations(query, [calc]) do - {:ok, [{calc, expression}]} -> - {:cont, - {:ok, - [ - {%{ - calc - | module: Ash.Resource.Calculation.Expression, - opts: [expr: expression] - }, direction} - | sort - ]}} + end) + |> Enum.reduce_while({:ok, []}, fn + {%Ash.Resource.Calculation{} = resource_calculation, direction}, {:ok, sort} -> + case Ash.Query.Calculation.from_resource_calculation( + query.resource, + resource_calculation, + source_context: query.context + ) do + {:ok, calc} -> + case hydrate_calculations(query, [calc]) do + {:ok, [{calc, expression}]} -> + {:cont, + {:ok, + [ + {%{ + calc + | module: Ash.Resource.Calculation.Expression, + opts: [expr: expression] + }, direction} + | sort + ]}} - {:error, error} -> - {:halt, {:error, error}} - end + {:error, error} -> + {:halt, {:error, error}} + end - {:error, error} -> - {:halt, {:error, error}} - end + {:error, error} -> + {:halt, {:error, error}} + end - {%Ash.Query.Calculation{} = calc, direction}, {:ok, sort} -> - case hydrate_calculations(query, [calc]) do - {:ok, [{calc, expression}]} -> - {:cont, - {:ok, - [ - {%{ - calc - | module: Ash.Resource.Calculation.Expression, - opts: [expr: expression] - }, direction} - | sort - ]}} + {%Ash.Query.Calculation{} = calc, direction}, {:ok, sort} -> + case hydrate_calculations(query, [calc]) do + {:ok, [{calc, expression}]} -> + {:cont, + {:ok, + [ + {%{ + calc + | module: Ash.Resource.Calculation.Expression, + opts: [expr: expression] + }, direction} + | sort + ]}} - {:error, error} -> - {:halt, {:error, error}} - end + {:error, error} -> + {:halt, {:error, error}} + end - {%Ash.Resource.Aggregate{} = agg, direction}, {:ok, sort} -> - case query_aggregate_from_resource_aggregate(query, agg) do - {:ok, agg} -> {:cont, {:ok, [{agg, direction} | sort]}} - {:error, error} -> {:halt, {:error, error}} - end + {%Ash.Resource.Aggregate{} = agg, direction}, {:ok, sort} -> + case query_aggregate_from_resource_aggregate(query, agg) do + {:ok, agg} -> {:cont, {:ok, [{agg, direction} | sort]}} + {:error, error} -> {:halt, {:error, error}} + end - {other, direction}, {:ok, sort} -> - {:cont, {:ok, [{other, direction} | sort]}} - end) - |> case do - {:ok, sort} -> - sort = - Enum.map(sort, fn {field, direction} -> - case field do - %struct{} = field - when struct in [ - Ash.Query.Calculation, - Ash.Aggregate.Calculation, - Ash.Resource.Calculation, - Ash.Resource.Aggregate - ] -> - {add_calc_context( - field, - actor, - authorize?, - tenant, - tracer, - domain, - query.resource, - parent_stack: parent_stack_from_context(query.context), - source_context: query.context - ), direction} - - field -> - {field, direction} - end - end) + {other, direction}, {:ok, sort} -> + {:cont, {:ok, [{other, direction} | sort]}} + end) + |> case do + {:ok, sort} -> + sort = + Enum.map(sort, fn {field, direction} -> + case field do + %struct{} = field + when struct in [ + Ash.Query.Calculation, + Ash.Aggregate.Calculation, + Ash.Resource.Calculation, + Ash.Resource.Aggregate + ] -> + {add_calc_context( + field, + actor, + authorize?, + tenant, + tracer, + domain, + query.resource, + parent_stack: parent_stack_from_context(query.context), + source_context: query.context + ), direction} + + field -> + {field, direction} + end + end) - {:ok, %{query | sort: Enum.reverse(sort)}} + {:ok, %{query | key => Enum.reverse(sort)}} - {:error, error} -> - {:error, error} - end + {:error, error} -> + {:error, error} + end + end) end defp hydrate_aggregates(query) do @@ -1781,8 +1832,14 @@ defmodule Ash.Actions.Read do expr expr -> - {:ok, expr} = Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) - expr + if calc.type do + {:ok, expr} = + Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) + + expr + else + expr + end end {:ok, expr} = @@ -1833,11 +1890,11 @@ defmodule Ash.Actions.Read do end) end - defp add_calc_context_to_sort(%{sort: empty}, _, _, _, _, _, _, _opts) when empty in [[], nil], + defp add_calc_context_to_sort(empty, _, _, _, _, _, _, _opts) when empty in [[], nil], do: {:ok, empty} - defp add_calc_context_to_sort(query, actor, authorize?, tenant, tracer, resource, domain, opts) do - query.sort + defp add_calc_context_to_sort(sort, actor, authorize?, tenant, tracer, resource, domain, opts) do + sort |> Enum.reduce_while({:ok, []}, fn {%struct{} = calc, order}, {:ok, acc} when struct in [ @@ -1849,7 +1906,7 @@ defmodule Ash.Actions.Read do calc = add_calc_context(calc, actor, authorize?, tenant, tracer, domain, resource, opts) if should_expand_expression?(struct, calc, opts) do - case expand_expression(calc, resource, opts[:parent_stack]) do + case expand_expression(calc, resource, opts[:parent_stack], opts[:first_combination]) do {:ok, expr} -> args = case calc do @@ -1863,7 +1920,7 @@ defmodule Ash.Actions.Read do actor: actor, tenant: tenant, args: args, - context: query.context + context: opts[:source_context] ) expr = @@ -1904,7 +1961,7 @@ defmodule Ash.Actions.Read do calc.module.has_expression?() end - defp expand_expression(calc, resource, parent_stack) do + defp expand_expression(calc, resource, parent_stack, first_combination) do calc.module.expression(calc.opts, calc.context) |> case do %Ash.Query.Function.Type{} = expr -> @@ -1914,10 +1971,21 @@ defmodule Ash.Actions.Read do expr expr -> - {:ok, expr} = Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) - expr + if calc.type do + {:ok, expr} = + Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) + + expr + else + expr + end end - |> Ash.Filter.hydrate_refs(%{resource: resource, public?: false, parent_stack: parent_stack}) + |> Ash.Filter.hydrate_refs(%{ + resource: resource, + public?: false, + parent_stack: parent_stack, + first_combination: first_combination + }) end @doc false @@ -2354,19 +2422,50 @@ defmodule Ash.Actions.Read do def add_calc_context_to_query(query, actor, authorize?, tenant, tracer, domain, opts) do {:ok, sort} = add_calc_context_to_sort( - query, + query.sort, actor, authorize?, tenant, tracer, query.resource, domain, - opts + Keyword.put(opts, :first_combination, Enum.at(query.combination_of, 0)) ) %{ query | sort: sort, + combination_of: + Enum.map(query.combination_of, fn + %Ash.Query.Combination{} = combination -> + {:ok, sort} = + add_calc_context_to_sort( + combination.sort, + actor, + authorize?, + tenant, + tracer, + query.resource, + domain, + opts + ) + + %{ + combination + | filter: + add_calc_context_to_filter( + combination.filter, + actor, + authorize?, + tenant, + tracer, + domain, + query.resource, + opts + ), + sort: sort + } + end), aggregates: Map.new(query.aggregates, fn {key, agg} -> {key, @@ -3349,6 +3448,7 @@ defmodule Ash.Actions.Read do case Ash.Filter.hydrate_refs(expression, %{ resource: query.resource, public?: false, + first_combination: Enum.at(query.combination_of, 0), parent_stack: parent_stack_from_context(query.context) }) do {:ok, expression} -> @@ -3691,8 +3791,14 @@ defmodule Ash.Actions.Read do expr expr -> - {:ok, expr} = Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) - expr + if calc.type do + {:ok, expr} = + Ash.Query.Function.Type.new([expr, calc.type, calc.constraints]) + + expr + else + expr + end end {:ok, expr} = diff --git a/lib/ash/actions/sort.ex b/lib/ash/actions/sort.ex index bb0f882d2..a505f974f 100644 --- a/lib/ash/actions/sort.ex +++ b/lib/ash/actions/sort.ex @@ -69,33 +69,53 @@ defmodule Ash.Actions.Sort do * `:resource` - The resource being sorted. """ def runtime_sort(results, sort, opts \\ []) - def runtime_sort([], _empty, _), do: [] - def runtime_sort(results, empty, _) when empty in [nil, []], do: results - def runtime_sort([single_result], _, _), do: [single_result] - def runtime_sort(results, [{field, direction}], opts) do + def runtime_sort([], _sort, _opts) do + [] + end + + def runtime_sort(results, sort, opts) do resource = get_resource(results, opts) + sort = + if Keyword.get(opts, :rename_calcs?, true) do + sort + |> Enum.with_index() + |> Enum.map(fn + {{%dynamic{load: nil} = field, order}, index} + when dynamic in [Ash.Query.Calculation, Ash.Query.Aggregate] -> + {%{field | name: {:__ash_runtime_sort__, index}}, order} + + {other, _} -> + other + end) + else + sort + end + + results + |> do_runtime_sort(resource, sort, opts) + |> maybe_rekey(results, resource, Keyword.get(opts, :rekey?, true)) + end + + defp do_runtime_sort([], _resource, _empty, _), do: [] + defp do_runtime_sort(results, _resource, empty, _) when empty in [nil, []], do: results + defp do_runtime_sort([single_result], _resource, _, _), do: [single_result] + + defp do_runtime_sort(results, resource, [{field, direction}], opts) do results |> load_field(field, resource, opts) |> Enum.sort_by(&resolve_field(&1, field), to_sort_by_fun(direction)) end - def runtime_sort(results, [{field, direction} | rest], opts) do - # we need check if the field supports simple equality, and if so then we can use - # uniq_by - # - # otherwise, we need to do our own matching - resource = get_resource(results, opts) - + defp do_runtime_sort(results, resource, [{field, direction} | rest], opts) do results |> load_field(field, resource, opts) |> Enum.group_by(&resolve_field(&1, field)) |> Enum.sort_by(fn {key, _value} -> key end, to_sort_by_fun(direction)) |> Enum.flat_map(fn {_, records} -> - runtime_sort(records, rest, Keyword.put(opts, :rekey?, false)) + do_runtime_sort(records, resource, rest, Keyword.put(opts, :rekey?, false)) end) - |> maybe_rekey(results, resource, Keyword.get(opts, :rekey?, true)) end defp get_resource(results, opts) do @@ -116,54 +136,101 @@ defmodule Ash.Actions.Sort do end defp maybe_rekey(new_results, results, resource, true) do - Enum.map(new_results, fn new_result -> - Enum.find(results, fn result -> - resource.primary_key_matches?(new_result, result) + if Ash.Resource.Info.primary_key(resource) == [] do + new_results + else + Enum.map(new_results, fn new_result -> + Enum.find(results, new_result, fn result -> + resource.primary_key_matches?(new_result, result) + end) end) - end) + end end defp maybe_rekey(new_results, _, _, _), do: new_results def runtime_distinct(results, sort, opts \\ []) - def runtime_distinct([], _empty, _), do: [] - def runtime_distinct(results, empty, _) when empty in [nil, []], do: results - def runtime_distinct([single_result], _, _), do: [single_result] - - def runtime_distinct([%resource{} | _] = results, distinct, opts) do - # we need check if the field supports simple equality, and if so then we can use - # uniq_by - # - # otherwise, we need to do our own matching + + def runtime_distinct([], _sort, _opts) do + [] + end + + def runtime_distinct(results, sort, opts) do + resource = get_resource(results, opts) + + results + |> do_runtime_distinct(resource, sort, opts) + |> maybe_rekey(results, resource, Keyword.get(opts, :rekey?, true)) + end + + defp do_runtime_distinct([], _resource, _empty, _), do: [] + defp do_runtime_distinct(results, _resource, empty, _) when empty in [nil, []], do: results + defp do_runtime_distinct([single_result], _resource, _, _), do: [single_result] + + defp do_runtime_distinct(results, resource, distinct, opts) do + distinct = + distinct + |> Enum.with_index() + |> Enum.map(fn + {{%dynamic{load: nil} = field, order}, index} + when dynamic in [Ash.Query.Calculation, Ash.Query.Aggregate] -> + {%{field | name: {:__ash_runtime_sort__, index}}, order} + + {other, _} -> + other + end) + fields = Enum.map(distinct, &elem(&1, 0)) results - |> load_field(fields, resource, opts) - |> Enum.to_list() - |> runtime_sort(distinct, opts) - |> Enum.uniq_by(&Map.take(&1, fields)) + |> Stream.with_index() + |> Stream.map(fn {record, index} -> + Ash.Resource.set_metadata(record, %{__runtime_distinct_index__: index}) + end) + |> do_runtime_sort( + resource, + distinct, + Keyword.merge(opts, rename_calcs?: false, resource: resource) + ) + |> Stream.uniq_by(fn record -> + Enum.map(fields, fn field -> + resolve_field(record, field) + end) + end) + |> Enum.sort_by(& &1.__metadata__.__runtime_distinct_index__) end defp load_field(records, field, resource, opts) do + query = + resource + |> Ash.Query.select([]) + |> Ash.Query.load(field) + |> Ash.Query.set_context(%{private: %{internal?: true}}) + if is_nil(opts[:domain]) do records else - records - |> Stream.chunk_every(100) - |> Stream.flat_map(fn batch -> - query = - resource - |> Ash.Query.select([]) - |> Ash.Query.load(field) - |> Ash.Query.set_context(%{private: %{internal?: true}}) - - Ash.load!(batch, query, - domain: opts[:domain], - reuse_values?: true, - authorize?: false, - lazy?: opts[:lazy?] || false - ) - end) + if opts[:maybe_not_distinct?] do + Enum.map(records, fn record -> + Ash.load!(record, query, + domain: opts[:domain], + reuse_values?: true, + authorize?: false, + lazy?: opts[:lazy?] || false + ) + end) + else + records + |> Stream.chunk_every(100) + |> Stream.flat_map(fn batch -> + Ash.load!(batch, query, + domain: opts[:domain], + reuse_values?: true, + authorize?: false, + lazy?: opts[:lazy?] || false + ) + end) + end end end diff --git a/lib/ash/data_layer/data_layer.ex b/lib/ash/data_layer/data_layer.ex index 72b8a2c9b..8f26223d9 100644 --- a/lib/ash/data_layer/data_layer.ex +++ b/lib/ash/data_layer/data_layer.ex @@ -65,9 +65,13 @@ defmodule Ash.DataLayer do | %{required(:type) => :custom, required(:metadata) => map()} | %{required(:type) => atom, required(:metadata) => map()} + @type combination_type :: :union | :union_all | :intersection + @type feature() :: :transact | :multitenancy + | :combine + | {:combine, combination_type} | {:atomic, :update} | {:atomic, :upsert} | {:lateral_join, list(Ash.Resource.t())} @@ -106,6 +110,12 @@ defmodule Ash.DataLayer do {Ash.Resource.t(), atom, atom, Ash.Resource.Relationships.relationship()} @callback functions(Ash.Resource.t()) :: [module] + @callback combination_of( + combine :: [{combination_type, data_layer_query()}], + resource :: Ash.Resource.t(), + domain :: Ash.Domain.t() + ) :: + {:ok, data_layer_query()} | {:error, term} @callback filter(data_layer_query(), Ash.Filter.t(), resource :: Ash.Resource.t()) :: {:ok, data_layer_query()} | {:error, term} @callback sort(data_layer_query(), Ash.Sort.t(), resource :: Ash.Resource.t()) :: @@ -296,6 +306,7 @@ defmodule Ash.DataLayer do return_query: 2, lock: 3, run_query_with_lateral_join: 4, + combination_of: 3, create: 2, update: 2, set_context: 3, @@ -623,6 +634,34 @@ defmodule Ash.DataLayer do end end + @spec combination_of( + [{combination_type, data_layer_query}], + resource :: Ash.Resource.t(), + domain :: Ash.Domain.t() + ) :: + {:ok, data_layer_query()} | {:error, term} + def combination_of(combinations, resource, domain) do + data_layer = Ash.DataLayer.data_layer(resource) + + if data_layer.can?(resource, :combine) do + combinations + |> Enum.map(&elem(&1, 0)) + |> Enum.uniq() + |> Enum.find(fn type -> + !Ash.DataLayer.can?({:combine, type}, resource) + end) + |> case do + nil -> + data_layer.combination_of(combinations, resource, domain) + + type -> + {:error, "Data layer does not support combining queries with `#{inspect(type)}`"} + end + else + {:error, "Data layer does not support combining queries"} + end + end + @spec sort(data_layer_query(), Ash.Sort.t(), Ash.Resource.t()) :: {:ok, data_layer_query()} | {:error, term} def sort(query, sort, resource) do diff --git a/lib/ash/data_layer/ets/ets.ex b/lib/ash/data_layer/ets/ets.ex index 73225d087..eaca9e7f2 100644 --- a/lib/ash/data_layer/ets/ets.ex +++ b/lib/ash/data_layer/ets/ets.ex @@ -52,11 +52,13 @@ defmodule Ash.DataLayer.Ets do :resource, :filter, :limit, - :sort, :tenant, :domain, - :distinct, - :distinct_sort, + :select, + sort: [], + distinct: [], + distinct_sort: nil, + combination_of: [], context: %{}, calculations: [], aggregates: [], @@ -177,6 +179,8 @@ defmodule Ash.DataLayer.Ets do def can?(resource, :async_engine), do: not Ash.DataLayer.Ets.Info.private?(resource) def can?(_, {:lateral_join, _}), do: true def can?(_, :bulk_create), do: true + def can?(_, :combine), do: true + def can?(_, {:combine, _type}), do: true def can?(_, :composite_primary_key), do: true def can?(_, :expression_calculation), do: true def can?(_, :expression_calculation_sort), do: true @@ -293,6 +297,12 @@ defmodule Ash.DataLayer.Ets do {:ok, %{query | context: context}} end + @doc false + @impl true + def select(query, select, _resource) do + {:ok, %{query | select: select}} + end + @doc false @impl true def filter(query, filter, _resource) do @@ -369,27 +379,39 @@ defmodule Ash.DataLayer.Ets do end end + @doc false + @impl true + def combination_of(combinations, resource, domain) do + {:ok, %Query{combination_of: combinations, resource: resource, domain: domain}} + end + @doc false @impl true def run_query( %Query{ resource: resource, - filter: filter, - offset: offset, - limit: limit, - sort: sort, - distinct: distinct, - distinct_sort: distinct_sort, - tenant: tenant, - calculations: calculations, - aggregates: aggregates, - domain: domain, - context: context - }, + combination_of: combination_of, + tenant: tenant + } = query, _resource, parent \\ nil ) do - with {:ok, records} <- get_records(resource, tenant), + maybe_not_distinct? = Enum.any?(combination_of, &(elem(&1, 0) == :union_all)) + + with {:ok, records} when records != [] <- + get_records(resource, combination_of, parent, tenant), + %Query{ + filter: filter, + offset: offset, + limit: limit, + sort: sort, + distinct: distinct, + distinct_sort: distinct_sort, + calculations: calculations, + aggregates: aggregates, + domain: domain, + context: context + } <- load_combinations(query), {:ok, records} <- filter_matches( records, @@ -399,9 +421,8 @@ defmodule Ash.DataLayer.Ets do context[:private][:actor], parent ), - records <- Sort.runtime_sort(records, distinct_sort || sort, domain: domain), - records <- Sort.runtime_distinct(records, distinct, domain: domain), - records <- Sort.runtime_sort(records, sort, domain: domain), + records <- + distinct_and_sort(records, sort, distinct, distinct_sort, maybe_not_distinct?, domain), records <- Enum.drop(records, offset || []), records <- do_limit(records, limit), {:ok, records} <- @@ -415,11 +436,54 @@ defmodule Ash.DataLayer.Ets do ) do {:ok, records} else + {:ok, records} -> + {:ok, records} + {:error, error} -> {:error, error} end end + defp distinct_and_sort(records, sort, distinct, distinct_sort, maybe_not_distinct?, domain) do + if distinct in [nil, []] do + Sort.runtime_sort(records, sort, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + else + if distinct_sort in [nil, []] do + Sort.runtime_sort(records, sort, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + |> Sort.runtime_distinct(distinct, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + else + records + |> Sort.runtime_sort(distinct_sort, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + |> Sort.runtime_distinct(distinct, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + |> Sort.runtime_sort(sort, + domain: domain, + maybe_not_distinct?: maybe_not_distinct?, + rekey?: false + ) + end + end + end + defp do_limit(records, nil), do: records defp do_limit(records, limit), do: Enum.take(records, limit) @@ -740,7 +804,7 @@ defmodule Ash.DataLayer.Ets do context[:tenant], context[:actor] ), - sorted <- Sort.runtime_sort(filtered, query.sort, domain: domain) do + sorted <- Sort.runtime_sort(filtered, query.sort, domain: domain, rekey?: false) do field = field || Enum.at(Ash.Resource.Info.primary_key(query.resource), 0) value = @@ -966,7 +1030,7 @@ defmodule Ash.DataLayer.Ets do Map.get(record, name) end - defp get_records(resource, tenant) do + defp get_records(resource, [], _parent, tenant) do with {:ok, table} <- wrap_or_create_table(resource, tenant), {:ok, record_tuples} <- ETS.Set.to_list(table), records <- Enum.map(record_tuples, &elem(&1, 1)) do @@ -974,6 +1038,211 @@ defmodule Ash.DataLayer.Ets do end end + defp get_records(resource, [{_, first} | _] = combinations_of, parent, tenant) do + field_set = + Enum.uniq_by( + Enum.map( + first.select || + MapSet.to_list(Ash.Resource.Info.selected_by_default_attribute_names(resource)), + &Ash.Resource.Info.attribute(resource, &1) + ) ++ + Enum.map(first.calculations, &elem(&1, 0)), + & &1.name + ) + + {simple_equality, non_simple_equality} = + Enum.split_with(field_set, fn %{type: type} -> + is_nil(type) || Ash.Type.simple_equality?(type) + end) + + {base, matcher, grouper} = + case {simple_equality, non_simple_equality} do + {[], non_simple_equality} -> + non_simple_equality = Enum.map(non_simple_equality, & &1.name) + base = [] + + matcher = fn record, acc -> + Enum.any?(acc, fn existing -> + Enum.all?(non_simple_equality, fn %{type: type, name: name} -> + Ash.Type.equal?(type, combo_field(record, name), combo_field(existing, name)) + end) + end) + end + + grouper = fn records, acc -> + Enum.concat(acc, records) + end + + {base, matcher, grouper} + + {simple_equality, []} -> + simple_equality_names = Enum.map(simple_equality, & &1.name) + base = MapSet.new() + matcher = fn record, acc -> combo_fields(record, simple_equality_names) in acc end + + grouper = fn records, acc -> + Enum.reduce(records, acc, fn record, acc -> + MapSet.put(acc, combo_fields(record, simple_equality_names)) + end) + end + + {base, matcher, grouper} + + {simple_equality, non_simple_equality} -> + simple_equality_names = Enum.map(simple_equality, & &1.name) + non_simple_equality_names = Enum.map(non_simple_equality, & &1.name) + base = Map.new() + + matcher = fn record, acc -> + key = combo_fields(record, simple_equality_names) + + case Map.fetch(acc, key) do + {:ok, non_simple_equality_values} -> + Enum.all?(non_simple_equality, fn %{type: type, name: name} -> + Ash.Type.equal?( + type, + combo_field(record, name), + Map.get(non_simple_equality_values, name) + ) + end) + + :error -> + false + end + end + + grouper = fn records, acc -> + Enum.reduce(records, acc, fn record, acc -> + Map.put( + acc, + combo_fields(record, simple_equality_names), + combo_fields(record, non_simple_equality_names) + ) + end) + end + + {base, matcher, grouper} + end + + combinations_of + |> Enum.reduce_while({:ok, [], base}, fn {type, combination}, {:ok, records, acc} -> + case run_query( + %{combination | tenant: tenant}, + resource, + parent + ) do + {:ok, results} -> + case type do + type when type in [:base, :union_all] -> + {:cont, {:ok, [records, results], grouper.(results, acc)}} + + :union -> + new_results = + Enum.reject(results, fn result -> + matcher.(result, acc) + end) + + {:cont, {:ok, [records, new_results], grouper.(new_results, acc)}} + + :intersect -> + temp_results_acc = + results + |> Enum.filter(fn result -> + matcher.(result, acc) + end) + |> grouper.(base) + + records = + records + |> Stream.flat_map(&List.flatten(List.wrap(&1))) + |> Enum.filter(fn result -> + matcher.(result, temp_results_acc) + end) + + {:cont, {:ok, records, grouper.(records, acc)}} + + :except -> + temp_results_acc = grouper.(results, base) + + results = + records + |> Stream.flat_map(&List.flatten(List.wrap(&1))) + |> Enum.reject(fn result -> + matcher.(result, temp_results_acc) + end) + + {:cont, {:ok, results, grouper.(results, base)}} + end + + {:error, error} -> + {:halt, {:error, error}} + end + end) + |> case do + {:ok, records, _acc} -> + records + |> List.flatten() + |> then(&{:ok, &1}) + + {:error, error} -> + {:error, error} + end + end + + defp load_combinations(query) do + %{ + query + | filter: do_load_combinations(query.filter), + sort: + Enum.map(query.sort, fn + {%Ash.Query.Calculation{ + module: Ash.Resource.Calculation.Expression, + opts: opts + } = calc, order} -> + {%{calc | opts: Keyword.update!(opts, :expr, &do_load_combinations/1)}, order} + + other -> + other + end), + distinct: + Enum.map(query.distinct, fn + {%Ash.Query.Calculation{ + module: Ash.Resource.Calculation.Expression, + opts: opts + } = calc, order} -> + {%{calc | opts: Keyword.update!(opts, :expr, &do_load_combinations/1)}, order} + + other -> + other + end), + distinct_sort: + query.distinct_sort && + Enum.map(query.distinct_sort, fn + {%Ash.Query.Calculation{ + module: Ash.Resource.Calculation.Expression, + opts: opts + } = calc, order} -> + {%{calc | opts: Keyword.update!(opts, :expr, &do_load_combinations/1)}, order} + + other -> + other + end) + } + end + + defp do_load_combinations(filter) do + Ash.Filter.map(filter, fn + %Ash.Query.Ref{ + combinations?: true, + attribute: %{load?: false} = attr + } = ref -> + %{ref | attribute: %{attr | load?: true}} + + other -> + other + end) + end + @doc false def cast_records(records, resource) do records @@ -995,6 +1264,16 @@ defmodule Ash.DataLayer.Ets do end end + defp combo_field(record, field) do + Map.get(record.calculations, field, Map.get(record, field)) + end + + defp combo_fields(record, fields) do + Map.new(fields, fn field -> + {field, combo_field(record, field)} + end) + end + @doc false def cast_record(record, resource) do resource diff --git a/lib/ash/expr/expr.ex b/lib/ash/expr/expr.ex index e5a3b3019..de2cf3a61 100644 --- a/lib/ash/expr/expr.ex +++ b/lib/ash/expr/expr.ex @@ -44,6 +44,7 @@ defmodule Ash.Expr do def expr?({:_actor, _}), do: true def expr?({:_arg, _}), do: true def expr?({:_ref, _, _}), do: true + def expr?({:_combinations, _}), do: true def expr?({:_parent, _, _}), do: true def expr?({:_parent, _}), do: true def expr?({:_atomic_ref, _}), do: true @@ -95,6 +96,9 @@ defmodule Ash.Expr do @doc "A template helper for creating a reference to a related path" def ref(path, name) when is_list(path) and is_atom(name), do: {:_ref, path, name} + @doc "A template helper for creating a reference" + def combinations(name) when is_atom(name), do: {:_combinations, name} + @doc "A template helper for creating a parent reference" def parent(expr), do: {:_parent, [], expr} @@ -144,6 +148,43 @@ defmodule Ash.Expr do end end + @doc """ + Creates an expression calculation for use in sort and distinct statements. + + ## Examples + + ```elixir + Ash.Query.sort(query, [ + {calc(string_upcase(name), :asc}, + {calc(count_nils([field1, field2]), type: :integer), :desc}) + ]) + ``` + """ + @spec calc(Macro.t(), opts :: Keyword.t()) :: t() + defmacro calc(expression, opts \\ []) do + quote generated: true do + require Ash.Expr + opts = unquote(opts) + type = opts[:type] && Ash.Type.get_type(opts[:type]) + constraints = opts[:constraints] || [] + name = opts[:name] || :__calc__ + + case Ash.Query.Calculation.new( + name, + Ash.Resource.Calculation.Expression, + [expr: Ash.Expr.expr(unquote(expression))], + type, + constraints + ) do + {:ok, calc} -> calc + {:error, term} -> raise Ash.Error.to_ash_error(term) + end + end + end + + @doc """ + Creates an expression. See the [Expressions guide](/documentation/topics/reference/expressions.md) for more. + """ @spec expr(Macro.t()) :: t() defmacro expr(do: body) do quote location: :keep do @@ -215,6 +256,12 @@ defmodule Ash.Expr do fill_template(path, Keyword.take(opts, [:actor, :tenant, :args, :context])) } + {:_combinations, name} -> + %Ash.Query.Ref{ + attribute: fill_template(name, Keyword.take(opts, [:actor, :tenant, :args, :context])), + combinations?: true + } + other -> other end) diff --git a/lib/ash/filter/filter.ex b/lib/ash/filter/filter.ex index 2931e55e3..a845c48dd 100644 --- a/lib/ash/filter/filter.ex +++ b/lib/ash/filter/filter.ex @@ -2291,6 +2291,7 @@ defmodule Ash.Filter do expression |> do_list_refs(no_longer_simple?, in_an_eq?, expand_calculations?, expand_get_path?) |> Enum.uniq() + |> Enum.reject(& &1.combinations?) end defp do_list_refs(list, no_longer_simple?, in_an_eq?, expand_calculations?, expand_get_path?) @@ -3537,6 +3538,19 @@ defmodule Ash.Filter do ) end + def do_hydrate_refs({:_combinations, value}, context) do + do_hydrate_refs( + %Ash.Query.Ref{ + attribute: value, + relationship_path: [], + input?: Map.get(context, :input?, false), + combinations?: true, + resource: context.root_resource + }, + context + ) + end + def do_hydrate_refs({:_ref, path, value}, context) do do_hydrate_refs( %Ash.Query.Ref{ @@ -3559,6 +3573,24 @@ defmodule Ash.Filter do end end + def do_hydrate_refs(%Ref{combinations?: true, attribute: attribute} = ref, %{ + resource: resource, + first_combination: first_combination + }) + when is_atom(attribute) and not is_nil(first_combination) do + with :error <- combination_calc(first_combination, attribute), + :error <- combination_attr(resource, first_combination, attribute) do + {:error, "Invalid combination reference #{inspect(ref)}"} + else + {:error, error} -> {:error, error} + {:ok, attr} -> {:ok, %{ref | attribute: attr}} + end + end + + def do_hydrate_refs(%Ref{combinations?: true} = ref, _) do + {:ok, ref} + end + def do_hydrate_refs( %Ref{relationship_path: relationship_path, resource: nil} = ref, %{resource: resource} = context @@ -3893,6 +3925,48 @@ defmodule Ash.Filter do {:ok, val} end + defp combination_calc(first_combination, attribute) do + with {:ok, calc} <- Map.fetch(first_combination.calculations, attribute) do + case calc do + %Ash.Query.Calculation{type: type, constraints: constraints} when not is_nil(type) -> + {:ok, %Ash.Query.CombinationAttr{type: type, constraints: constraints, name: attribute}} + + %Ash.Query.Calculation{} = calculation -> + {:error, + """ + Calculation #{inspect(calculation)} must be added with a type to refer to it with `combinations/1` + + For example: + + calc(foo <> bar, type: :string) + """} + + other -> + {:error, "Invalid combination calculation #{inspect(other)}"} + end + end + end + + defp combination_attr(resource, first_combination, attribute_name) do + attribute = Ash.Resource.Info.attribute(resource, attribute_name) + + if attribute do + if (first_combination.select && attribute_name in first_combination.select) || + attribute_name in Ash.Resource.Info.selected_by_default_attribute_names(resource) do + {:ok, + %Ash.Query.CombinationAttr{ + type: attribute.type, + constraints: attribute.constraints, + name: attribute.name + }} + else + {:error, "#{attribute_name} is not selected nor is it a calculation in the combinations."} + end + else + :error + end + end + defp validate_data_layers_support_boolean_filters(%BooleanExpression{ op: :or, left: left, diff --git a/lib/ash/filter/runtime.ex b/lib/ash/filter/runtime.ex index 86766db4e..6ed8dc806 100644 --- a/lib/ash/filter/runtime.ex +++ b/lib/ash/filter/runtime.ex @@ -1,16 +1,7 @@ defmodule Ash.Filter.Runtime do @moduledoc """ - Checks a record to see if it matches a filter statement. - - We can't always tell if a record matches a filter statement, and as such this - function may return `:unknown`. Additionally, some expressions wouldn't ever - make sense outside of the context of the data layer, and will always be an - error. For example, if you used the trigram search features in - `ash_postgres`. That logic would need to be handwritten in Elixir and would - need to be a *perfect* copy of the postgres implementation. That isn't a - realistic goal. This generally should not affect anyone using the standard - framework features, but if you were to attempt to use this module with a data - layer like `ash_postgres`, certain expressions will behave unpredictably. + Tools to checks a record to see if it matches a filter statement, or to + evalute expressions against records. """ alias Ash.Query.{BooleanExpression, Not, Ref} @@ -373,6 +364,7 @@ defmodule Ash.Filter.Runtime do defp resolve_expr({:_arg, _}, _, _, _, _), do: :unknown defp resolve_expr({:_ref, _}, _, _, _, _), do: :unknown defp resolve_expr({:_ref, _, _}, _, _, _, _), do: :unknown + defp resolve_expr({:_combinations, _}, _, _, _, _), do: :unknown defp resolve_expr({:_parent, _}, _, _, _, _), do: :unknown defp resolve_expr({:_parent, _, _}, _, _, _, _), do: :unknown defp resolve_expr({:_atomic_ref, _}, _, _, _, _), do: :unknown @@ -818,6 +810,7 @@ defmodule Ash.Filter.Runtime do %Ash.Query.Ref{ relationship_path: relationship_path, resource: resource, + combinations?: combinations?, attribute: %Ash.Query.Calculation{ module: module, opts: opts, @@ -830,7 +823,8 @@ defmodule Ash.Filter.Runtime do parent, _resource, unknown_on_unknown_refs? - ) do + ) + when combinations? != true do result = record |> get_related(relationship_path, unknown_on_unknown_refs?) @@ -940,6 +934,29 @@ defmodule Ash.Filter.Runtime do end end + defp resolve_ref(%Ash.Query.Ref{combinations?: true, attribute: %{load?: false}}, _, _, _, true) do + :unknown + end + + defp resolve_ref( + %Ash.Query.Ref{attribute: %{name: name}, combinations?: true}, + record, + _, + _, + unknown_on_unknown_refs? + ) do + with :error <- Map.fetch(record.calculations, name), + :error <- Map.fetch(record, name) do + if unknown_on_unknown_refs? do + :unknown + else + {:ok, nil} + end + else + {:ok, value} -> {:ok, value} + end + end + defp resolve_ref( %Ash.Query.Ref{ relationship_path: [], diff --git a/lib/ash/policy/authorizer/authorizer.ex b/lib/ash/policy/authorizer/authorizer.ex index 34652e529..0668e7ada 100644 --- a/lib/ash/policy/authorizer/authorizer.ex +++ b/lib/ash/policy/authorizer/authorizer.ex @@ -931,14 +931,22 @@ defmodule Ash.Policy.Authorizer do } end + {type, constraints} = + case type do + {:array, _} -> {type, []} + {type, constraints} -> {type, constraints} + type -> {type, nil} + end + expr = - Ash.Sort.expr_sort( + Ash.Expr.calc( if ^expr do ^field else nil end, - type + type: type, + constraints: constraints ) {{expr, data}, acc} @@ -1843,8 +1851,23 @@ defmodule Ash.Policy.Authorizer do {:ok, {Ash.Policy.Check.Expression, expr: expr}} end + @templates [ + :_actor, + :_arg, + :_ref, + :_combination, + :_parent, + :_atomic_ref, + :_context + ] + def template_var({template_var, _} = expr) - when template_var in [:_actor, :_arg, :_ref, :_parent, :_atomic_ref, :_context] do + when template_var in @templates do + {:ok, {Ash.Policy.Check.Expression, expr: expr}} + end + + def template_var({template_var, _, _} = expr) + when template_var in @templates do {:ok, {Ash.Policy.Check.Expression, expr: expr}} end diff --git a/lib/ash/query/combination.ex b/lib/ash/query/combination.ex new file mode 100644 index 000000000..8646faf45 --- /dev/null +++ b/lib/ash/query/combination.ex @@ -0,0 +1,113 @@ +defmodule Ash.Query.Combination do + @moduledoc """ + Represents one combination in a combination of queries. + """ + + @type t :: %__MODULE__{ + filter: Ash.Expr.t(), + sort: Ash.Sort.t(), + limit: pos_integer() | nil, + offset: pos_integer() | nil, + select: [atom], + calculations: %{atom() => Ash.Query.Calculation.t()}, + type: :base | :union | :union_all | :except | :intersect + } + + defstruct [:filter, :sort, :limit, :offset, :select, calculations: %{}, type: :base] + + @doc """ + The initial combination of a combined query. + """ + def base(opts) do + %{struct(__MODULE__, opts) | type: :base} + end + + @doc """ + Unions the query with the previous combinations, discarding duplicates when all fields are equal. + """ + def union(opts) do + %{struct(__MODULE__, opts) | type: :union} + end + + @doc """ + Unions the query with the previous combinations, keeping all rows. + """ + def union_all(opts) do + %{struct(__MODULE__, opts) | type: :union_all} + end + + @doc """ + Removes all rows that are present in the previous combinations *and* this one. + """ + def except(opts) do + %{struct(__MODULE__, opts) | type: :except} + end + + @doc """ + Intersects the query with the previous combinations, keeping only rows that are present in the previous combinations and this one. + """ + def intersect(opts) do + %{struct(__MODULE__, opts) | type: :intersect} + end + + defimpl Inspect do + import Inspect.Algebra + + def inspect(combination, opts) do + if opts.custom_options[:in_query?] && combination.type != :base do + sort? = combination.sort != [] + filter? = !!combination.filter + limit? = !!combination.limit + offset? = !!combination.offset + select? = !!combination.select + calculations? = !Enum.empty?(combination.calculations) + + container_doc( + "#{combination.type} #Ash.Query.Combination<", + [ + or_empty(concat("filter: ", to_doc(combination.filter, opts)), filter?), + or_empty(concat("sort: ", to_doc(combination.sort, opts)), sort?), + or_empty(concat("offset: ", to_doc(combination.offset, opts)), offset?), + or_empty(concat("limit: ", to_doc(combination.limit, opts)), limit?), + or_empty(concat("select: ", to_doc(combination.select, opts)), select?), + or_empty( + concat("calculations: ", to_doc(combination.calculations, opts)), + calculations? + ) + ], + ">", + opts, + fn str, _ -> str end + ) + else + sort? = combination.sort != [] + filter? = !!combination.filter + limit? = !!combination.limit + offset? = !!combination.offset + select? = !!combination.select + calculations? = !Enum.empty?(combination.calculations) + + container_doc( + "#Ash.Query.Combination<", + [ + or_empty(concat("filter: ", to_doc(combination.filter, opts)), filter?), + or_empty(concat("sort: ", to_doc(combination.sort, opts)), sort?), + or_empty(concat("offset: ", to_doc(combination.offset, opts)), offset?), + or_empty(concat("limit: ", to_doc(combination.limit, opts)), limit?), + or_empty(concat("select: ", to_doc(combination.select, opts)), select?), + or_empty( + concat("calculations: ", to_doc(combination.calculations, opts)), + calculations? + ) + ], + ">", + opts, + fn str, _ -> str end + ) + end + end + + defp or_empty(value, true), do: value + defp or_empty(_, false), do: empty() + end +end diff --git a/lib/ash/query/combination_attr.ex b/lib/ash/query/combination_attr.ex new file mode 100644 index 000000000..9b35bb8ca --- /dev/null +++ b/lib/ash/query/combination_attr.ex @@ -0,0 +1,4 @@ +defmodule Ash.Query.CombinationAttr do + @moduledoc false + defstruct [:name, :type, :constraints, load?: false] +end diff --git a/lib/ash/query/query.ex b/lib/ash/query/query.ex index 8b7d8981a..43a585360 100644 --- a/lib/ash/query/query.ex +++ b/lib/ash/query/query.ex @@ -169,6 +169,8 @@ defmodule Ash.Query do invalid_keys: MapSet.new(), load_through: %{}, action_failed?: false, + combination_of: [], + combination_calcs: [], after_action: [], authorize_results: [], aggregates: %{}, @@ -197,6 +199,8 @@ defmodule Ash.Query do filter: Ash.Filter.t() | nil, resource: module, tenant: term(), + combination_of: [Ash.Query.Combination.t()], + combination_calcs: list(atom), timeout: pos_integer() | nil, action_failed?: boolean, after_action: [ @@ -302,6 +306,7 @@ defmodule Ash.Query do distinct? = query.distinct not in [[], nil] lock? = not is_nil(query.lock) page? = not is_nil(query.page) + combination_of? = query.combination_of != [] container_doc( "#Ash.Query<", @@ -309,6 +314,14 @@ defmodule Ash.Query do concat("resource: ", inspect(query.resource)), or_empty(concat("tenant: ", to_doc(query.to_tenant, opts)), tenant?), arguments(query, opts), + # TODO: inspect these specially + or_empty( + concat( + "combination_of: ", + to_doc(query.combination_of, %{opts | custom_options: [in_query?: true]}) + ), + combination_of? + ), or_empty(concat("filter: ", to_doc(query.filter, opts)), filter?), or_empty(concat("sort: ", to_doc(query.sort, opts)), sort?), or_empty(concat("distinct_sort: ", to_doc(query.distinct_sort, opts)), distinct_sort?), @@ -426,6 +439,92 @@ defmodule Ash.Query do end end + @doc """ + Produces a query that is the combination of multiple queries. + + All aspects of the parent query are applied to the combination in total. + + See `Ash.Query.Combination` for more on creating combination queries. + + ### Example + + ```elixir + # Top ten users not on a losing streak and top ten users who are not on a winning streak + User + |> Ash.Query.filter(active == true) + |> Ash.Query.combination_of([ + # must always begin with a base combination + Ash.Query.Combination.base( + sort: [score: :desc], + filter: expr(not(on_a_losing_streak)), + limit: 10 + ), + Ash.Query.Combination.union( + sort: [score: :asc], + filter: expr(not(on_a_winning_streak)), + limit: 10 + ) + ]) + |> Ash.read!() + ``` + + ### Select and calculations + + There is no `select` available for combinations, instead the select of the outer query + is used for each combination. However, you can use the `calculations` field in + `Ash.Query.Combination` to add expression calculations. Those calculations can "overwrite" + a selected attribute, or can introduce a new field. Note that, for SQL data layers, all + combinations will be required to have the same number of fields in their SELECT statement, + which means that if one combination adds a calculation, all of the others must also add + that calculation. + + In this example, we compute separate match scores + + ```elixir + query = "fred" + + User + |> Ash.Query.filter(active == true) + |> Ash.Query.combination_of([ + # must always begin with a base combination + Ash.Query.Combination.base( + filter: expr(trigram_similarity(user_name, ^query) >= 0.5), + calculate: %{ + match_score: trigram_similarity(user_name, ^query) + }, + sort: [ + calc(trigram_similarity(user_name, ^query), :desc) + ], + limit: 10 + ), + Ash.Query.Combination.union( + filter: expr(trigram_similarity(email, ^query) >= 0.5), + calculate: %{ + match_score: trigram_similarity(email, ^query) + }, + sort: [ + calc(trigram_similarity(email, ^query), :desc) + ], + limit: 10 + ) + ]) + |> Ash.read!() + ``` + """ + @spec combination_of(t(), Ash.Query.Combination.t()) :: t() + def combination_of(query, combinations) do + query = new(query) + + %{query | combination_of: query.combination_of ++ List.wrap(combinations)} + end + + @spec combination_calcs(t(), list(atom)) :: t() + def combination_calcs(query, fields) do + query = new(query) + + %{query | combination_calcs: fields} + end + @doc """ Attach a sort statement to the query labelled as user input. @@ -487,27 +586,32 @@ defmodule Ash.Query do add_error(query, :sort, "Data layer does not support sorting") end end - |> sequence_expr_sorts() + |> sequence_sorts() end - # sobelow_skip ["DOS.BinToAtom", "DOS.StringToAtom"] - defp sequence_expr_sorts(%{sort: sort} = query) when is_list(sort) and sort != [] do + defp sequence_sorts(query) do %{ query - | sort: - query.sort - |> Enum.with_index() - |> Enum.map(fn - {{%Ash.Query.Calculation{name: :__expr_sort__} = field, direction}, index} -> - {%{field | name: String.to_atom("__expr_sort__#{index}"), load: nil}, direction} - - {other, _} -> - other - end) + | sort: sequence_sort(query.sort), + distinct_sort: sequence_sort(query.distinct_sort), + distinct: sequence_sort(query.distinct) } end - defp sequence_expr_sorts(query), do: query + defp sequence_sort(nil), do: nil + + # sobelow_skip ["DOS.BinToAtom", "DOS.StringToAtom"] + defp sequence_sort(statement) do + statement + |> Enum.with_index() + |> Enum.map(fn + {{%Ash.Query.Calculation{name: :__calc__} = field, direction}, index} -> + {%{field | name: String.to_atom("__calc__#{index}"), load: nil}, direction} + + {other, _} -> + other + end) + end @doc """ Attach a filter statement to the query. @@ -2597,7 +2701,11 @@ defmodule Ash.Query do {module, opts} = case module_and_opts do {module, opts} -> - {module, opts} + if Ash.Expr.expr?({module, opts}) do + {Ash.Resource.Calculation.Expression, expr: {module, opts}} + else + {module, opts} + end module when is_atom(module) -> {module, []} @@ -3022,16 +3130,16 @@ defmodule Ash.Query do ## Expression Sorts - You can use `Ash.Expr.expr_sort/2-3` to sort on expressions: + You can use the `Ash.Expr.calc/2` macro to sort on expressions: ```elixir - # Sort on an expression import Ash.Expr - Ash.Query.sort(query, expr_sort(count(friends), :desc)) + + # Sort on an expression + Ash.Query.sort(query, calc(count(friends), :desc)) # Specify a type (required in some cases when we can't determine a type) - import Ash.Expr - Ash.Query.sort(query, expr_sort(fragment("some_sql(?)", field), :desc, :string)) + Ash.Query.sort(query, [{calc(fragment("some_sql(?)", field, type: :string), :desc}]) ``` ## Sort Strings @@ -3129,7 +3237,7 @@ defmodule Ash.Query do add_error(query, :sort, "Data layer does not support sorting") end end - |> sequence_expr_sorts() + |> sequence_sorts() end @doc """ @@ -3284,19 +3392,34 @@ defmodule Ash.Query do end def data_layer_query(%{resource: resource, domain: domain} = ash_query, opts) do - query = opts[:initial_query] || Ash.DataLayer.resource_to_query(resource, domain) - context = ash_query.context |> Map.put(:action, ash_query.action) |> Map.put_new(:private, %{}) |> put_in([:private, :tenant], ash_query.tenant) + |> Map.put_new(:data_layer, %{}) + + context = + if opts[:previous_combination] do + Map.update!( + context, + :data_layer, + &Map.put(&1, :previous_combination, opts[:previous_combination]) + ) + else + context + end - with {:ok, query} <- + with {:ok, query, new_context} <- initial_data_layer_query(ash_query, domain, opts), + {:ok, query} <- Ash.DataLayer.set_context( resource, query, - context + Map.update!( + context, + :data_layer, + &Map.merge(&1, new_context) + ) ), {:ok, query} <- add_tenant(query, ash_query), {:ok, query} <- Ash.DataLayer.select(query, ash_query.select, ash_query.resource), @@ -3336,6 +3459,76 @@ defmodule Ash.Query do end end + defp initial_data_layer_query(ash_query, domain, opts) do + cond do + opts[:initial_query] -> + {:ok, opts[:initial_query], %{}} + + ash_query.combination_of != [] -> + case combination_queries(ash_query) do + {:ok, combinations, previous} -> + with {:ok, query} <- + Ash.DataLayer.combination_of(combinations, ash_query.resource, domain) do + {:ok, query, %{previous_combination: previous, combination_of_queries?: true}} + end + + {:error, error} -> + {:error, error} + end + + true -> + {:ok, opts[:initial_query] || Ash.DataLayer.resource_to_query(ash_query.resource, domain), + %{}} + end + end + + defp combination_queries(query) do + base_query = Ash.Query.new(query.resource) + + Enum.reduce_while( + query.combination_of, + {:ok, [], nil}, + fn combination, {:ok, combinations, previous} -> + calculations = + Enum.map(combination.calculations, fn {name, calc} -> + {%{calc | name: name, load: nil}, calc.module.expression(calc.opts, calc.context)} + end) + + base_query + |> limit(combination.limit) + |> offset(combination.offset) + |> do_filter(combination.filter) + |> sort(combination.sort) + |> Ash.Query.set_context(query.context) + |> Ash.Query.set_context(%{data_layer: %{union_query?: true}}) + |> then(fn + %{valid?: true} = combination_query -> + case data_layer_query(combination_query, + previous_combination: previous, + data_layer_calculations: calculations + ) do + {:ok, combination_query} -> + {:cont, + {:ok, [{combination.type, combination_query} | combinations], combination_query}} + + {:error, error} -> + {:halt, {:error, error}} + end + + %{valid?: false, errors: errors} -> + {:halt, {:error, errors}} + end) + end + ) + |> then(fn + {:ok, unions, previous} -> + {:ok, Enum.reverse(unions), previous} + + {:error, error} -> + {:error, error} + end) + end + defp maybe_return_query(query, resource, opts) do if Keyword.get(opts, :run_return_query?, true) do Ash.DataLayer.return_query(query, resource) diff --git a/lib/ash/query/ref.ex b/lib/ash/query/ref.ex index bec2e5e07..b3ff918b8 100644 --- a/lib/ash/query/ref.ex +++ b/lib/ash/query/ref.ex @@ -1,6 +1,14 @@ defmodule Ash.Query.Ref do @moduledoc "Represents a relation/attribute reference" - defstruct [:attribute, :relationship_path, :resource, :simple_equality?, :bare?, :input?] + defstruct [ + :attribute, + :resource, + :simple_equality?, + :bare?, + :input?, + combinations?: false, + relationship_path: [] + ] @doc "Returns the referenced field" def name(%__MODULE__{attribute: %{name: name}}), do: name @@ -45,6 +53,13 @@ defmodule Ash.Query.Ref do end end) <> "." <> "#{name}" end + |> then(fn value -> + if ref.combinations? do + "combinations(#{value})" + else + value + end + end) end end end diff --git a/lib/ash/resource.ex b/lib/ash/resource.ex index 6eaf92764..dd757c2dd 100644 --- a/lib/ash/resource.ex +++ b/lib/ash/resource.ex @@ -143,8 +143,27 @@ defmodule Ash.Resource do @persist {:domain, domain} end + if embedded? do + @persist {:embedded?, true} + + require Ash.EmbeddableType + + Ash.EmbeddableType.define_embeddable_type(embed_nil_values?: embed_nil_values?) + end + end + end + + @doc false + # sobelow_skip ["DOS.StringToAtom"] + @impl Spark.Dsl + def handle_before_compile(_opts) do + quote do + require Ash.Schema + + @derive_inspect_for_redacted_fields false + Ash.Schema.define_schema() + if Ash.Schema.define?(__MODULE__) do - @derive_inspect_for_redacted_fields false module = __MODULE__ defimpl Inspect do @@ -198,25 +217,6 @@ defmodule Ash.Resource do end end - if embedded? do - @persist {:embedded?, true} - - require Ash.EmbeddableType - - Ash.EmbeddableType.define_embeddable_type(embed_nil_values?: embed_nil_values?) - end - end - end - - @doc false - # sobelow_skip ["DOS.StringToAtom"] - @impl Spark.Dsl - def handle_before_compile(_opts) do - quote do - require Ash.Schema - - Ash.Schema.define_schema() - @all_arguments __MODULE__ |> Ash.Resource.Info.actions() |> Enum.flat_map(& &1.arguments) diff --git a/lib/ash/sort/sort.ex b/lib/ash/sort/sort.ex index e562ae559..1aa1192e1 100644 --- a/lib/ash/sort/sort.ex +++ b/lib/ash/sort/sort.ex @@ -22,7 +22,7 @@ defmodule Ash.Sort do alias Ash.Error.Query.{InvalidSortOrder, NoSuchField} @doc """ - Builds an expression to be used in a sort statement. + Builds an expression to be used in a sort statement. Prefer to use `Ash.Expr.calc/2` instead. For example: @@ -36,6 +36,7 @@ defmodule Ash.Sort do defmacro expr_sort(expression, type \\ nil) do quote generated: true do require Ash.Expr + type = unquote(type) {type, constraints} = @@ -53,18 +54,11 @@ defmodule Ash.Sort do {nil, []} end - type = type && Ash.Type.get_type(type) - - case Ash.Query.Calculation.new( - :__expr_sort__, - Ash.Resource.Calculation.Expression, - [expr: Ash.Expr.expr(unquote(expression))], - type, - constraints - ) do - {:ok, calc} -> calc - {:error, term} -> raise Ash.Error.to_ash_error(term) - end + Ash.Expr.calc(unquote(expression), + type: type, + constraints: constraints, + name: :__calc__ + ) end end @@ -274,7 +268,7 @@ defmodule Ash.Sort do } Ash.Query.Calculation.new( - :__expr_sort__, + :__calc__, Ash.Resource.Calculation.Expression, [expr: ref], type, @@ -289,7 +283,7 @@ defmodule Ash.Sort do } Ash.Query.Calculation.new( - :__expr_sort__, + :__calc__, Ash.Resource.Calculation.Expression, [expr: ref], type, @@ -318,7 +312,7 @@ defmodule Ash.Sort do %Ash.Resource.Calculation{} = calc -> with {:ok, calc} <- Ash.Query.Calculation.from_resource_calculation(resource, calc, args: input) do - {:ok, %{calc | name: :__expr_sort__}} + {:ok, %{calc | name: :__calc__}} end nil -> @@ -342,7 +336,7 @@ defmodule Ash.Sort do %Ash.Resource.Calculation{} = calc -> with {:ok, calc} <- Ash.Query.Calculation.from_resource_calculation(resource, calc, args: input) do - {:ok, %{calc | name: :__expr_sort__}} + {:ok, %{calc | name: :__calc__}} end nil -> @@ -547,5 +541,5 @@ defmodule Ash.Sort do collation that affects their sorting, making it unpredictable from the perspective of a tool using the database: https://www.postgresql.org/docs/current/collation.html """ - defdelegate runtime_sort(results, sort, domain \\ nil), to: Ash.Actions.Sort + defdelegate runtime_sort(results, sort, opts \\ []), to: Ash.Actions.Sort end diff --git a/mix.exs b/mix.exs index eed609e5e..c42613252 100644 --- a/mix.exs +++ b/mix.exs @@ -82,6 +82,7 @@ defmodule Ash.MixProject do "documentation/topics/advanced/reactor.md", "documentation/topics/advanced/monitoring.md", "documentation/topics/advanced/pagination.livemd", + "documentation/topics/advanced/combination-queries.md", "documentation/topics/advanced/timeouts.md", "documentation/topics/advanced/multitenancy.md", "documentation/topics/advanced/writing-extensions.md", @@ -356,7 +357,8 @@ defmodule Ash.MixProject do # DSLs {:spark, "~> 2.1 and >= 2.2.29"}, # Ash resources are backed by ecto scheams - {:ecto, "~> 3.7"}, + # {:ecto, "~> 3.7"}, + {:ecto, path: "../../oss/ecto", override: true}, # Used by the ETS data layer {:ets, "~> 0.8"}, # Data & types diff --git a/test/query_test.exs b/test/query_test.exs index 890d01176..432dc3c67 100644 --- a/test/query_test.exs +++ b/test/query_test.exs @@ -3,6 +3,7 @@ defmodule Ash.Test.QueryTest do use ExUnit.Case, async: true require Ash.Query + import Ash.Expr alias Ash.Test.Domain, as: Domain @@ -101,6 +102,245 @@ defmodule Ash.Test.QueryTest do end end + describe "union" do + test "it combines multiple queries into one result set" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "fred", email: "b@bar.com"}) + Ash.create!(User, %{name: "fred", email: "c@bar.com"}) + Ash.create!(User, %{name: "fred", email: "d@baz.com"}) + Ash.create!(User, %{name: "george"}) + + assert [%User{email: "c@bar.com"}, %User{email: "a@bar.com"}] = + User + |> Ash.Query.filter(name == "fred") + |> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(contains(email, "bar.com")), + limit: 1, + sort: [email: :desc] + ), + Ash.Query.Combination.union_all( + filter: expr(contains(email, "bar.com")), + limit: 1, + sort: [email: :asc] + ) + ]) + |> Ash.read!() + end + + test "you can define computed properties" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "fred", email: "b@bar.com"}) + Ash.create!(User, %{name: "fred", email: "c@bar.com"}) + Ash.create!(User, %{name: "fred", email: "d@baz.com"}) + Ash.create!(User, %{name: "george"}) + + assert [%User{email: "c@bar.com", calculations: %{match_group: 1}}] = + User + |> Ash.Query.filter(name == "fred") + |> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(contains(email, "bar.com")), + limit: 1, + calculations: %{ + match_group: calc(1, type: :integer), + same_thing: calc(1, type: :integer) + }, + sort: [email: :desc] + ), + Ash.Query.Combination.union_all( + filter: expr(contains(email, "bar.com")), + calculations: %{ + match_group: calc(2, type: :integer), + same_thing: calc(1, type: :integer) + }, + limit: 1, + sort: [email: :asc] + ) + ]) + |> Ash.Query.distinct_sort([{calc(^combinations(:same_thing)), :asc}]) + |> Ash.Query.sort([{calc(^combinations(:match_group)), :desc}]) + |> Ash.Query.distinct([{calc(^combinations(:same_thing)), :asc}]) + |> Ash.Query.calculate(:match_group, :integer, expr(^combinations(:match_group))) + |> Ash.read!() + end + + test "it handles combinations with intersect" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "john", email: "j@bar.com"}) + Ash.create!(User, %{name: "fred", email: "f@baz.com"}) + Ash.create!(User, %{name: "alice", email: "a@bar.com"}) + + assert [%User{name: "fred", email: "a@bar.com"}] = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base(filter: expr(name == "fred")), + Ash.Query.Combination.intersect(filter: expr(contains(email, "bar.com"))) + ]) + |> Ash.read!() + end + + test "it handles combinations with except" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "fred", email: "b@bar.com"}) + Ash.create!(User, %{name: "john", email: "j@baz.com"}) + + result = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base(filter: expr(name == "fred")), + Ash.Query.Combination.except(filter: expr(contains(email, "b@"))) + ]) + |> Ash.read!() + + assert length(result) == 1 + assert hd(result).email == "a@bar.com" + end + + test "combinations with multiple union_all" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "alice", email: "a@baz.com"}) + Ash.create!(User, %{name: "john", email: "j@qux.com"}) + + result = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base(filter: expr(name == "fred")), + Ash.Query.Combination.union_all(filter: expr(name == "alice")), + Ash.Query.Combination.union_all(filter: expr(name == "john")) + ]) + |> Ash.read!() + + assert length(result) == 3 + assert Enum.any?(result, &(&1.name == "fred")) + assert Enum.any?(result, &(&1.name == "alice")) + assert Enum.any?(result, &(&1.name == "john")) + end + + test "combination with offset" do + # Create users with ascending email for predictable sort order + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "fred", email: "b@bar.com"}) + Ash.create!(User, %{name: "fred", email: "c@bar.com"}) + + result = + User + |> Ash.Query.filter(name == "fred") + |> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(contains(email, "bar.com")), + offset: 1, + limit: 2, + sort: [email: :asc] + ) + ]) + |> Ash.read!() + + assert length(result) == 2 + assert hd(result).email == "b@bar.com" + assert List.last(result).email == "c@bar.com" + end + + test "combinations with complex calculations" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "john", email: "j@baz.com"}) + + result = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(name == "fred"), + calculations: %{ + domain: calc("bar", type: :string), + full_name: calc(name <> "@bar", type: :string) + } + ), + Ash.Query.Combination.union_all( + filter: expr(name == "john"), + calculations: %{ + domain: calc("baz", type: :string), + full_name: calc(name <> "@baz", type: :string) + } + ) + ]) + |> Ash.Query.calculate(:email_domain, :string, expr(^combinations(:domain))) + |> Ash.Query.calculate(:display_name, :string, expr(^combinations(:full_name))) + |> Ash.read!() + + fred = Enum.find(result, &(&1.name == "fred")) + john = Enum.find(result, &(&1.name == "john")) + + assert fred.calculations.email_domain == "bar" + assert fred.calculations.display_name == "fred@bar" + assert john.calculations.email_domain == "baz" + assert john.calculations.display_name == "john@baz" + end + + test "combinations with sorting by calculation" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + Ash.create!(User, %{name: "alice", email: "a@baz.com"}) + Ash.create!(User, %{name: "john", email: "j@qux.com"}) + + result = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base(calculations: %{sort_order: calc(3, type: :integer)}), + Ash.Query.Combination.union_all( + filter: expr(name == "alice"), + calculations: %{sort_order: calc(1, type: :integer)} + ), + Ash.Query.Combination.union_all( + filter: expr(name == "john"), + calculations: %{sort_order: calc(2, type: :integer)} + ) + ]) + |> Ash.Query.sort([{calc(^combinations(:sort_order)), :asc}, {:name, :asc}]) + |> Ash.Query.distinct(:name) + |> Ash.read!() + + assert [first, second, third | _] = result + assert first.name == "alice" + assert second.name == "john" + assert third.name == "fred" + end + + test "combination with distinct" do + Ash.create!(User, %{name: "fred", email: "a@bar.com"}) + # Same email domain + Ash.create!(User, %{name: "alice", email: "a@bar.com"}) + Ash.create!(User, %{name: "john", email: "j@baz.com"}) + + result = + User + |> Ash.Query.combination_of([ + Ash.Query.Combination.base( + filter: expr(contains(email, "bar.com")), + select: [:id], + calculations: %{email_domain: calc("bar.com", type: :string)} + ), + Ash.Query.Combination.union_all( + filter: expr(contains(email, "baz.com")), + select: [:id], + calculations: %{email_domain: calc("baz.com", type: :string)} + ) + ]) + |> Ash.Query.distinct([{calc(^combinations(:email_domain)), :asc}]) + |> Ash.read!() + + # Should only have 2 results since we're distinct on email domain + assert length(result) == 2 + + domains = + Enum.map(result, fn r -> + Map.get(r.calculations || %{}, :email_domain) + end) + |> Enum.reject(&is_nil/1) + + assert "bar.com" in domains + assert "baz.com" in domains + end + end + describe "filter" do test "can filter by list" do list = ["a", "b", "c"] diff --git a/test/type/union_test.exs b/test/type/union_test.exs index 0656de4f7..6951dfd66 100644 --- a/test/type/union_test.exs +++ b/test/type/union_test.exs @@ -1,4 +1,4 @@ -defmodule Ash.Test.Type.UnionTest do +defmodule Ash.Test.Filter.UnionTest do @moduledoc false use ExUnit.Case, async: true