Shield: Country Rules (#3828)

* Migration: add country rules

* Add CountryRule schema

* Implement CountryRule cache

* Add country rules context interface

* Start country rules cache

* Lookup country rules on ingestion

* Remove :shields feature flag from test helpers

* Add nested sidebar menu for Shields

* Fix typo

* IP Rules: hide description on mobile view

* Prepare SiteController to handle multiple shield types

* Seed some country shield

* Implement LV for country rules

* Remove "YOU" indicator from country rules

* Fix small build

* Format

* Update typespecs

* Make docs link point at /countries

* Fix flash on top of modal for Safari

* Build the rule struct with site_id provided up-front

* Clarify why we're messaging the ComboBox component

* Re-open combobox suggestions after pressing Escape

* Update changelog

* Fix font size in country table cells

* Pass `added_by` via rule add options

* Display site's timezone timestamps in rule tooltips

* Display formatted timestamps in site's timezone

And simplify+test Timezone module; an input timestamp converted
to UTC can never be ambiguous.

* Remove no-op atom

* Display the maximum number of rules when reached

* Improve readability of remove button tests

* Credo

---------

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>

CHANGELOG.md

@@ -2,6 +2,7 @@
 All notable changes to this project will be documented in this file.
 
 ### Added
+- County Block List in Site Settings
 - Query the `views_per_visit` metric based on imported data as well if possible
 - Group `operating_system_versions` by `operating_system` in Stats API breakdown
 - Add `operating_system_versions.csv` into the CSV export

assets/js/liveview/combo-box.js

@@ -18,8 +18,10 @@ export default (id) => ({
     this.selectionInProgress = false;
   },
   open() {
-    this.initFocus()
-    this.isOpen = true
+    if (!this.isOpen) {
+      this.initFocus()
+      this.isOpen = true
+    }
   },
   suggestionsCount() {
     return this.$refs.suggestions?.querySelectorAll('li').length
@@ -65,15 +67,15 @@ export default (id) => ({
   focusNext() {
     const nextIndex = this.nextFocusableIndex()
 
-    if (!this.isOpen) this.open()
+    this.open()
 
     this.setFocus(nextIndex)
     this.scrollTo(nextIndex)
   },
   focusPrev() {
     const prevIndex = this.prevFocusableIndex()
 
-    if (!this.isOpen) this.open()
+    this.open()
 
     this.setFocus(prevIndex)
     this.scrollTo(prevIndex)

lib/plausible/application.ex

@@ -59,6 +59,21 @@ defmodule Plausible.Application do
          interval: :timer.seconds(35),
          warmer_fn: :refresh_updated_recently
        ]},
+      {Plausible.Shield.CountryRuleCache, []},
+      {Plausible.Cache.Warmer,
+       [
+         child_name: Plausible.Shield.CountryRuleCache.All,
+         cache_impl: Plausible.Shield.CountryRuleCache,
+         interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
+         warmer_fn: :refresh_all
+       ]},
+      {Plausible.Cache.Warmer,
+       [
+         child_name: Plausible.Shield.CountryRuleCache.RecentlyUpdated,
+         cache_impl: Plausible.Shield.CountryRuleCache,
+         interval: :timer.seconds(35),
+         warmer_fn: :refresh_updated_recently
+       ]},
       {Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
       PlausibleWeb.Endpoint,
       {Oban, Application.get_env(:plausible, Oban)},

lib/plausible/ingestion/event.ex

@@ -27,6 +27,7 @@ defmodule Plausible.Ingestion.Event do
           | :invalid
           | :dc_ip
           | :site_ip_blocklist
+          | :site_country_blocklist
 
   @type t() :: %__MODULE__{
           domain: String.t() | nil,
@@ -97,11 +98,12 @@ defmodule Plausible.Ingestion.Event do
     [
       &drop_datacenter_ip/1,
       &drop_shield_rule_ip/1,
+      &put_geolocation/1,
+      &drop_shield_rule_country/1,
       &put_user_agent/1,
       &put_basic_info/1,
       &put_referrer/1,
       &put_utm_tags/1,
-      &put_geolocation/1,
       &put_props/1,
       &put_revenue/1,
       &put_salts/1,
@@ -232,6 +234,21 @@ defmodule Plausible.Ingestion.Event do
     end
   end
 
+  defp drop_shield_rule_country(
+         %__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event
+       )
+       when is_binary(domain) and is_binary(cc) do
+    case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
+      %Plausible.Shield.CountryRule{action: :deny} ->
+        drop(event, :site_country_blocklist)
+
+      _ ->
+        event
+    end
+  end
+
+  defp drop_shield_rule_country(%__MODULE__{} = event), do: event
+
   defp put_props(%__MODULE__{request: %{props: %{} = props}} = event) do
     # defensive: ensuring the keys/values are always in the same order
     {keys, values} = Enum.unzip(props)

lib/plausible/shield/country_rule.ex

@@ -0,0 +1,38 @@
+defmodule Plausible.Shield.CountryRule do
+  @moduledoc """
+  Schema for Country Block List
+  """
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @type t() :: %__MODULE__{}
+
+  @primary_key {:id, :binary_id, autogenerate: true}
+  schema "shield_rules_country" do
+    belongs_to :site, Plausible.Site
+    field :country_code, :string
+    field :action, Ecto.Enum, values: [:deny, :allow], default: :deny
+    field :added_by, :string
+
+    # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.Country.Cache`
+    field :from_cache?, :boolean, virtual: true, default: false
+    timestamps()
+  end
+
+  def changeset(rule, attrs) do
+    rule
+    |> cast(attrs, [:site_id, :country_code])
+    |> validate_required([:site_id, :country_code])
+    |> validate_length(:country_code, is: 2)
+    |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc ->
+      if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do
+        []
+      else
+        [country_code: "is invalid"]
+      end
+    end)
+    |> unique_constraint(:country_code,
+      name: :shield_rules_country_site_id_country_code_index
+    )
+  end
+end

lib/plausible/shield/country_rule_cache.ex

@@ -0,0 +1,65 @@
+defmodule Plausible.Shield.CountryRuleCache do
+  @moduledoc """
+  Allows retrieving Country Rules by domain and country code
+  """
+  alias Plausible.Shield.CountryRule
+
+  import Ecto.Query
+  use Plausible.Cache
+
+  @cache_name :country_blocklist_by_domain
+
+  @cached_schema_fields ~w(
+    id
+    country_code
+    action
+  )a
+
+  @impl true
+  def name(), do: @cache_name
+
+  @impl true
+  def child_id(), do: :cachex_country_blocklist
+
+  @impl true
+  def count_all() do
+    Plausible.Repo.aggregate(CountryRule, :count)
+  end
+
+  @impl true
+  def base_db_query() do
+    from rule in CountryRule,
+      inner_join: s in assoc(rule, :site),
+      select: {
+        s.domain,
+        s.domain_changed_from,
+        %{struct(rule, ^@cached_schema_fields) | from_cache?: true}
+      }
+  end
+
+  @impl true
+  def get_from_source({domain, country_code}) do
+    query =
+      base_db_query()
+      |> where([rule, site], rule.country_code == ^country_code and site.domain == ^domain)
+
+    case Plausible.Repo.one(query) do
+      {_, _, rule} -> %CountryRule{rule | from_cache?: false}
+      _any -> nil
+    end
+  end
+
+  @impl true
+  def unwrap_cache_keys(items) do
+    Enum.reduce(items, [], fn
+      {domain, nil, object}, acc ->
+        [{{domain, String.upcase(object.country_code)}, object} | acc]
+
+      {domain, domain_changed_from, object}, acc ->
+        [
+          {{domain, String.upcase(object.country_code)}, object},
+          {{domain_changed_from, String.upcase(object.country_code)}, object} | acc
+        ]
+    end)
+  end
+end

lib/plausible/shield/ip_rule.ex

@@ -22,7 +22,7 @@ defmodule Plausible.Shield.IPRule do
 
   def changeset(rule, attrs) do
     rule
-    |> cast(attrs, [:site_id, :inet, :description, :added_by])
+    |> cast(attrs, [:site_id, :inet, :description])
     |> validate_required([:site_id, :inet])
     |> disallow_netmask(:inet)
     |> unique_constraint(:inet,

lib/plausible/shields.ex

@@ -5,38 +5,93 @@ defmodule Plausible.Shields do
   import Ecto.Query
   alias Plausible.Repo
   alias Plausible.Shield
+  alias Plausible.Site
 
   @maximum_ip_rules 30
   def maximum_ip_rules(), do: @maximum_ip_rules
 
-  @spec list_ip_rules(Plausible.Site.t() | non_neg_integer()) :: [Shield.IPRule.t()]
-  def list_ip_rules(site_id) when is_integer(site_id) do
+  @maximum_country_rules 30
+  def maximum_country_rules(), do: @maximum_country_rules
+
+  @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()]
+  def list_ip_rules(site_or_id) do
+    list(Shield.IPRule, site_or_id)
+  end
+
+  @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
+          {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()}
+  def add_ip_rule(site_or_id, params, opts \\ []) do
+    opts =
+      Keyword.put(opts, :limit, {:inet, @maximum_ip_rules})
+
+    add(Shield.IPRule, site_or_id, params, opts)
+  end
+
+  @spec remove_ip_rule(Site.t() | non_neg_integer(), String.t()) :: :ok
+  def remove_ip_rule(site_or_id, rule_id) do
+    remove(Shield.IPRule, site_or_id, rule_id)
+  end
+
+  @spec count_ip_rules(Site.t() | non_neg_integer()) :: non_neg_integer()
+  def count_ip_rules(site_or_id) do
+    count(Shield.IPRule, site_or_id)
+  end
+
+  @spec list_country_rules(Site.t() | non_neg_integer()) :: [Shield.CountryRule.t()]
+  def list_country_rules(site_or_id) do
+    list(Shield.CountryRule, site_or_id)
+  end
+
+  @spec add_country_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
+          {:ok, Shield.CountryRule.t()} | {:error, Ecto.Changeset.t()}
+  def add_country_rule(site_or_id, params, opts \\ []) do
+    opts = Keyword.put(opts, :limit, {:country_code, @maximum_country_rules})
+    add(Shield.CountryRule, site_or_id, params, opts)
+  end
+
+  @spec remove_country_rule(Site.t() | non_neg_integer(), String.t()) :: :ok
+  def remove_country_rule(site_or_id, rule_id) do
+    remove(Shield.CountryRule, site_or_id, rule_id)
+  end
+
+  @spec count_country_rules(Site.t() | non_neg_integer()) :: non_neg_integer()
+  def count_country_rules(site_or_id) do
+    count(Shield.CountryRule, site_or_id)
+  end
+
+  defp list(schema, %Site{id: id}) do
+    list(schema, id)
+  end
+
+  defp list(schema, site_id) when is_integer(site_id) do
     Repo.all(
-      from r in Shield.IPRule,
+      from r in schema,
         where: r.site_id == ^site_id,
         order_by: [desc: r.inserted_at]
     )
   end
 
-  def list_ip_rules(%Plausible.Site{id: id}) do
-    list_ip_rules(id)
+  defp add(schema, %Site{id: id}, params, opts) do
+    add(schema, id, params, opts)
   end
 
-  @spec add_ip_rule(Plausible.Site.t() | non_neg_integer(), map()) ::
-          {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()}
-  def add_ip_rule(site_id, params) when is_integer(site_id) do
+  defp add(schema, site_id, params, opts) when is_integer(site_id) do
+    {field, max} = Keyword.fetch!(opts, :limit)
+
     Repo.transaction(fn ->
       result =
-        if count_ip_rules(site_id) >= @maximum_ip_rules do
+        if count(schema, site_id) >= max do
           changeset =
-            %Shield.IPRule{}
-            |> Shield.IPRule.changeset(Map.put(params, "site_id", site_id))
-            |> Ecto.Changeset.add_error(:inet, "maximum reached")
+            schema
+            |> struct(site_id: site_id)
+            |> schema.changeset(params)
+            |> Ecto.Changeset.add_error(field, "maximum reached")
 
           {:error, changeset}
         else
-          %Shield.IPRule{}
-          |> Shield.IPRule.changeset(Map.put(params, "site_id", site_id))
+          schema
+          |> struct(site_id: site_id, added_by: format_added_by(opts[:added_by]))
+          |> schema.changeset(params)
           |> Repo.insert()
         end
 
@@ -47,26 +102,23 @@ defmodule Plausible.Shields do
     end)
   end
 
-  def add_ip_rule(%Plausible.Site{id: id}, params) do
-    add_ip_rule(id, params)
+  defp remove(schema, %Site{id: id}, rule_id) do
+    remove(schema, id, rule_id)
   end
 
-  @spec remove_ip_rule(Plausible.Site.t() | non_neg_integer(), String.t()) :: :ok
-  def remove_ip_rule(site_id, rule_id) when is_integer(site_id) do
-    Repo.delete_all(from(r in Shield.IPRule, where: r.site_id == ^site_id and r.id == ^rule_id))
+  defp remove(schema, site_id, rule_id) when is_integer(site_id) do
+    Repo.delete_all(from(r in schema, where: r.site_id == ^site_id and r.id == ^rule_id))
     :ok
   end
 
-  def remove_ip_rule(%Plausible.Site{id: site_id}, rule_id) do
-    remove_ip_rule(site_id, rule_id)
+  defp count(schema, %Site{id: id}) do
+    count(schema, id)
   end
 
-  @spec count_ip_rules(Plausible.Site.t() | non_neg_integer()) :: non_neg_integer()
-  def count_ip_rules(site_id) when is_integer(site_id) do
-    Repo.aggregate(from(r in Shield.IPRule, where: r.site_id == ^site_id), :count)
+  defp count(schema, site_id) when is_integer(site_id) do
+    Repo.aggregate(from(r in schema, where: r.site_id == ^site_id), :count)
   end
 
-  def count_ip_rules(%Plausible.Site{id: id}) do
-    count_ip_rules(id)
-  end
+  defp format_added_by(nil), do: ""
+  defp format_added_by(%Plausible.Auth.User{} = user), do: "#{user.name} <#{user.email}>"
 end

lib/plausible/stats/breakdown.ex

@@ -34,14 +34,20 @@ defmodule Plausible.Stats.Breakdown do
 
     if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
 
+    no_revenue = {nil, metrics -- @revenue_metrics}
+
     {revenue_goals, metrics} =
-      if full_build?() && Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
-        revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
-        metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics
+      on_full_build do
+        if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
+          revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
+          metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics
 
-        {revenue_goals, metrics}
+          {revenue_goals, metrics}
+        else
+          no_revenue
+        end
       else
-        {nil, metrics -- @revenue_metrics}
+        no_revenue
       end
 
     metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics