Optimize usage per site lookup

lib/plausible/stats/clickhouse.ex

@@ -76,16 +76,31 @@ defmodule Plausible.Stats.Clickhouse do
 
   def usage_breakdown([], _date_range), do: {0, 0}
 
-  def per_site_usage_breakdown([], _date_range), do: []
+  def per_site_usage_breakdown(
+        site_ids,
+        date_range,
+        limit \\ Plausible.Teams.Billing.max_sites_for_usage_breakdown()
+      )
+
+  def per_site_usage_breakdown([], _date_range, _limit), do: []
 
-  def per_site_usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do
+  def per_site_usage_breakdown(
+        [sid | _] = site_ids,
+        date_range,
+        limit
+      )
+      when is_integer(sid) and length(site_ids) <= limit do
     ClickhouseRepo.all(
       from(e in "events_v2",
         where: e.site_id in ^site_ids,
         where: e.name != "engagement",
         where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
         where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
         group_by: e.site_id,
+        order_by: [
+          desc: fragment("countIf(? = 'pageview') + countIf(? != 'pageview')", e.name, e.name)
+        ],
+        limit: ^limit,
         select: {
           e.site_id,
           fragment("countIf(? = 'pageview')", e.name),

lib/plausible/teams/billing.ex

@@ -22,6 +22,7 @@ defmodule Plausible.Teams.Billing do
   @limit_sites_since ~D[2021-05-05]
 
   @max_sites_for_usage_breakdown 10
+  def max_sites_for_usage_breakdown, do: @max_sites_for_usage_breakdown
 
   @typep last_30_days_usage() :: %{:last_30_days => Quota.usage_cycle()}
   @typep monthly_pageview_usage() :: Quota.cycles_usage() | last_30_days_usage()
@@ -478,33 +479,47 @@ defmodule Plausible.Teams.Billing do
        do: []
 
   defp per_site_usage(owned_site_ids, date_range) do
-    site_domains =
-      Repo.all(
-        from(s in Plausible.Site,
-          where: s.id in ^owned_site_ids,
-          select: {s.id, s.domain}
+    per_site_usage_breakdown =
+      Plausible.Stats.Clickhouse.per_site_usage_breakdown(owned_site_ids, date_range)
+
+    active_site_ids = Enum.map(per_site_usage_breakdown, fn {site_id, _, _} -> site_id end)
+
+    domains =
+      Map.new(
+        Repo.all(
+          from(s in Plausible.Site,
+            where: s.id in ^active_site_ids,
+            select: {s.id, s.domain}
+          )
         )
       )
-      |> Map.new()
 
-    clickhouse_counts =
-      Plausible.Stats.Clickhouse.per_site_usage_breakdown(owned_site_ids, date_range)
-      |> Map.new(fn {site_id, pageviews, custom_events} ->
-        {site_id, {pageviews, custom_events}}
+    active_entries =
+      Enum.map(per_site_usage_breakdown, fn {site_id, pageviews, custom_events} ->
+        %{
+          domain: Map.get(domains, site_id),
+          pageviews: pageviews,
+          custom_events: custom_events,
+          total: pageviews + custom_events
+        }
       end)
 
-    site_domains
-    |> Enum.map(fn {site_id, domain} ->
-      {pageviews, custom_events} = Map.get(clickhouse_counts, site_id, {0, 0})
-
-      %{
-        domain: domain,
-        pageviews: pageviews,
-        custom_events: custom_events,
-        total: pageviews + custom_events
-      }
-    end)
-    |> Enum.sort_by(& &1.total, :desc)
+    zero_entries =
+      if length(per_site_usage_breakdown) < @max_sites_for_usage_breakdown do
+        inactive_site_ids = owned_site_ids -- active_site_ids
+
+        Repo.all(
+          from(s in Plausible.Site,
+            where: s.id in ^inactive_site_ids,
+            select: %{domain: s.domain, pageviews: 0, custom_events: 0, total: 0},
+            order_by: [asc: s.domain]
+          )
+        )
+      else
+        []
+      end
+
+    active_entries ++ zero_entries
   end
 
   @spec features_usage(Teams.Team.t() | nil, list() | nil) :: [atom()]