Allow domain change (#2803)

* Migration (PR: #2802)

* Implement Site.Domain interface allowing change and expiry

* Fixup seeds so they work with V2_MIGRATION_DONE=1

* Update Sites.Cache so it's capable of multi-keyed lookups

* Implement worker handling domain change expiration

* Implement domain change UI

* Implement transition period for public APIs

* Exclude v2 tests in primary test run

* Update lib/plausible_web/controllers/site_controller.ex

Co-authored-by: Vini Brasil <vini@hey.com>

* Update lib/plausible_web/controllers/site_controller.ex

Co-authored-by: Vini Brasil <vini@hey.com>

* Update moduledoc

* Update changelog

* Remove remnant from previous implementation attempt

* !fixup

* !fixup

* Implement domain change via Sites API

cc @ukutaht

* Update CHANGELOG

* Credo

* !fixup commit missing tests

* Allow continuous domain change within the same site

---------

Co-authored-by: Vini Brasil <vini@hey.com>

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 - 'Last updated X seconds ago' info to 'current visitors' tooltips
 - Add support for more Bamboo adapters, i.e. `Bamboo.MailgunAdapter`, `Bamboo.MandrillAdapter`, `Bamboo.SendGridAdapter` plausible/analytics#2649
+- Ability to change domain for existing site (requires numeric IDs data migration, instructions will be provided separately) UI + API (`PUT /api/v1/sites`)
 
 ### Fixed
 - Make goal-filtered CSV export return only unique_conversions timeseries in the 'visitors.csv' file

config/runtime.exs

@@ -363,7 +363,7 @@ end
 base_cron = [
   # Daily at midnight
   {"0 0 * * *", Plausible.Workers.RotateSalts},
-  #  hourly
+  # hourly
   {"0 * * * *", Plausible.Workers.ScheduleEmailReports},
   # hourly
   {"0 * * * *", Plausible.Workers.SendSiteSetupEmails},
@@ -374,7 +374,9 @@ base_cron = [
   # Every day at midnight
   {"0 0 * * *", Plausible.Workers.CleanEmailVerificationCodes},
   # Every day at 1am
-  {"0 1 * * *", Plausible.Workers.CleanInvitations}
+  {"0 1 * * *", Plausible.Workers.CleanInvitations},
+  # Every 2 hours
+  {"0 */2 * * *", Plausible.Workers.ExpireDomainChangeTransitions}
 ]
 
 cloud_cron = [
@@ -399,7 +401,8 @@ base_queues = [
   site_setup_emails: 1,
   clean_email_verification_codes: 1,
   clean_invitations: 1,
-  google_analytics_imports: 1
+  google_analytics_imports: 1,
+  domain_change_transition: 1
 ]
 
 cloud_queues = [

lib/plausible/site.ex

@@ -21,6 +21,9 @@ defmodule Plausible.Site do
     field :ingest_rate_limit_scale_seconds, :integer, default: 60
     field :ingest_rate_limit_threshold, :integer
 
+    field :domain_changed_from, :string
+    field :domain_changed_at, :naive_datetime
+
     embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
 
     many_to_many :members, User, join_through: Plausible.Site.Membership
@@ -40,21 +43,40 @@ defmodule Plausible.Site do
     timestamps()
   end
 
+  @domain_unique_error """
+  This domain cannot be registered. Perhaps one of your colleagues registered it? If that's not the case, please contact support@plausible.io
+  """
+
   def changeset(site, attrs \\ %{}) do
     site
     |> cast(attrs, [:domain, :timezone])
     |> clean_domain()
     |> validate_required([:domain, :timezone])
-    |> validate_format(:domain, ~r/^[-\.\\\/:\p{L}\d]*$/u,
-      message: "only letters, numbers, slashes and period allowed"
-    )
+    |> validate_domain_format()
     |> validate_domain_reserved_characters()
     |> unique_constraint(:domain,
-      message:
-        "This domain cannot be registered. Perhaps one of your colleagues registered it? If that's not the case, please contact support@plausible.io"
+      message: @domain_unique_error
     )
   end
 
+  def update_changeset(site, attrs \\ %{}, opts \\ []) do
+    at =
+      opts
+      |> Keyword.get(:at, NaiveDateTime.utc_now())
+      |> NaiveDateTime.truncate(:second)
+
+    attrs =
+      if Plausible.v2?() do
+        attrs
+      else
+        Map.delete(attrs, :domain)
+      end
+
+    site
+    |> changeset(attrs)
+    |> handle_domain_change(at)
+  end
+
   def crm_changeset(site, attrs) do
     site
     |> cast(attrs, [
@@ -183,9 +205,7 @@ defmodule Plausible.Site do
       |> String.replace_trailing("/", "")
       |> String.downcase()
 
-    change(changeset, %{
-      domain: clean_domain
-    })
+    change(changeset, %{domain: clean_domain})
   end
 
   # https://tools.ietf.org/html/rfc3986#section-2.2
@@ -203,4 +223,29 @@ defmodule Plausible.Site do
       changeset
     end
   end
+
+  defp validate_domain_format(changeset) do
+    validate_format(changeset, :domain, ~r/^[-\.\\\/:\p{L}\d]*$/u,
+      message: "only letters, numbers, slashes and period allowed"
+    )
+  end
+
+  defp handle_domain_change(changeset, at) do
+    new_domain = get_change(changeset, :domain)
+
+    if new_domain do
+      changeset
+      |> put_change(:domain_changed_from, changeset.data.domain)
+      |> put_change(:domain_changed_at, at)
+      |> unique_constraint(:domain,
+        name: "domain_change_disallowed",
+        message: @domain_unique_error
+      )
+      |> unique_constraint(:domain_changed_from,
+        message: @domain_unique_error
+      )
+    else
+      changeset
+    end
+  end
 end

lib/plausible/site/cache.ex

@@ -9,6 +9,10 @@ defmodule Plausible.Site.Cache do
   during tests via the `:sites_by_domain_cache_enabled` application env key.
   This can be overridden on case by case basis, using the child specs options.
 
+  NOTE: the cache allows lookups by both `domain` and `domain_changed_from`
+  fields - this is to allow traffic from sites whose domains changed within a certain
+  grace period (see: `Plausible.Site.Transfer`).
+
   When Cache is disabled via application env, the `get/1` function
   falls back to pure database lookups. This should help with introducing
   cached lookups in existing code, so that no existing tests should break.
@@ -49,6 +53,7 @@ defmodule Plausible.Site.Cache do
   @cached_schema_fields ~w(
      id
      domain
+     domain_changed_from
      ingest_rate_limit_scale_seconds
      ingest_rate_limit_threshold
    )a
@@ -91,6 +96,7 @@ defmodule Plausible.Site.Cache do
       from s in Site,
         select: {
           s.domain,
+          s.domain_changed_from,
           %{struct(s, ^@cached_schema_fields) | from_cache?: true}
         }
 
@@ -109,6 +115,7 @@ defmodule Plausible.Site.Cache do
         where: s.updated_at > ago(^15, "minute"),
         select: {
           s.domain,
+          s.domain_changed_from,
           %{struct(s, ^@cached_schema_fields) | from_cache?: true}
         }
 
@@ -124,6 +131,7 @@ defmodule Plausible.Site.Cache do
   def merge([], _), do: :ok
 
   def merge(new_items, opts) do
+    new_items = unwrap_cache_keys(new_items)
     cache_name = Keyword.get(opts, :cache_name, @cache_name)
     true = Cachex.put_many!(cache_name, new_items)
 
@@ -221,4 +229,14 @@ defmodule Plausible.Site.Cache do
     stop = System.monotonic_time()
     {stop - start, result}
   end
+
+  defp unwrap_cache_keys(items) do
+    Enum.reduce(items, [], fn
+      {domain, nil, object}, acc ->
+        [{domain, object} | acc]
+
+      {domain, domain_changed_from, object}, acc ->
+        [{domain, object}, {domain_changed_from, object} | acc]
+    end)
+  end
 end

lib/plausible/site/domain.ex

@@ -0,0 +1,62 @@
+defmodule Plausible.Site.Domain do
+  @expire_threshold_hours 72
+
+  @moduledoc """
+  Basic interface for domain changes.
+
+  Once `Plausible.DataMigration.NumericIDs` schema migration is ready, 
+  domain change operation will be enabled, accessible to the users.
+
+  We will set a transition period of #{@expire_threshold_hours} hours
+  during which, both old and new domains, will be accepted as traffic
+  identifiers to the same site. 
+
+  A periodic worker will call the `expire/0` function to end it where applicable.
+  See: `Plausible.Workers.ExpireDomainChangeTransitions`.
+
+  The underlying changeset for domain change (see: `Plausible.Site`) relies
+  on database trigger installed via `Plausible.Repo.Migrations.AllowDomainChange`
+  Postgres migration. The trigger checks if either `domain` or `domain_changed_from`
+  exist to ensure unicity.
+  """
+
+  alias Plausible.Site
+  alias Plausible.Repo
+
+  import Ecto.Query
+
+  @spec expire_change_transitions(integer()) :: {:ok, non_neg_integer()}
+  def expire_change_transitions(expire_threshold_hours \\ @expire_threshold_hours) do
+    {updated, _} =
+      Repo.update_all(
+        from(s in Site,
+          where: s.domain_changed_at < ago(^expire_threshold_hours, "hour")
+        ),
+        set: [
+          domain_changed_from: nil,
+          domain_changed_at: nil
+        ]
+      )
+
+    {:ok, updated}
+  end
+
+  @spec change(Site.t(), String.t(), Keyword.t()) ::
+          {:ok, Site.t()} | {:error, Ecto.Changeset.t()}
+  def change(%Site{} = site, new_domain, opts \\ []) do
+    changeset = Site.update_changeset(site, %{domain: new_domain}, opts)
+
+    changeset =
+      if Enum.empty?(changeset.changes) and is_nil(changeset.errors[:domain]) do
+        Ecto.Changeset.add_error(
+          changeset,
+          :domain,
+          "New domain must be different than the current one"
+        )
+      else
+        changeset
+      end
+
+    Repo.update(changeset)
+  end
+end

lib/plausible/sites.ex

@@ -107,7 +107,7 @@ defmodule Plausible.Sites do
       on: sm.site_id == s.id,
       where: sm.user_id == ^user_id,
       where: sm.role in ^roles,
-      where: s.domain == ^domain,
+      where: s.domain == ^domain or s.domain_changed_from == ^domain,
       select: s
     )
   end

lib/plausible_web/controllers/api/external_sites_controller.ex

@@ -49,6 +49,25 @@ defmodule PlausibleWeb.Api.ExternalSitesController do
     end
   end
 
+  def update_site(conn, %{"site_id" => site_id} = params) do
+    # for now this only allows to change the domain
+    site = Sites.get_for_user(conn.assigns[:current_user].id, site_id, [:owner, :admin])
+
+    if site && Plausible.v2?() do
+      case Plausible.Site.Domain.change(site, params["domain"]) do
+        {:ok, site} ->
+          json(conn, site)
+
+        {:error, changeset} ->
+          conn
+          |> put_status(400)
+          |> json(serialize_errors(changeset))
+      end
+    else
+      H.not_found(conn, "Site could not be found")
+    end
+  end
+
   defp expect_param_key(params, key) do
     case Map.fetch(params, key) do
       :error -> {:missing, key}

lib/plausible_web/controllers/site_controller.ex

@@ -329,11 +329,10 @@ defmodule PlausibleWeb.SiteController do
   end
 
   def update_settings(conn, %{"site" => site_params}) do
-    site = conn.assigns[:site]
-    changeset = site |> Plausible.Site.changeset(site_params)
-    res = changeset |> Repo.update()
+    site = conn.assigns[:site] |> Repo.preload(:custom_domain)
+    changeset = Plausible.Site.update_changeset(site, site_params)
 
-    case res do
+    case Repo.update(changeset) do
       {:ok, site} ->
         site_session_key = "authorized_site__" <> site.domain
 
@@ -343,7 +342,13 @@ defmodule PlausibleWeb.SiteController do
         |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))
 
       {:error, changeset} ->
-        render(conn, "settings_general.html", site: site, changeset: changeset)
+        conn
+        |> put_flash(:error, "Could not update your site settings")
+        |> render("settings_general.html",
+          site: site,
+          changeset: changeset,
+          layout: {PlausibleWeb.LayoutView, "site_settings.html"}
+        )
     end
   end
 
@@ -867,4 +872,49 @@ defmodule PlausibleWeb.SiteController do
         |> redirect(to: Routes.site_path(conn, :settings_general, site.domain))
     end
   end
+
+  def change_domain(conn, _params) do
+    if Plausible.v2?() do
+      changeset = Plausible.Site.update_changeset(conn.assigns.site)
+
+      render(conn, "change_domain.html",
+        changeset: changeset,
+        layout: {PlausibleWeb.LayoutView, "focus.html"}
+      )
+    else
+      render_error(conn, 404)
+    end
+  end
+
+  def change_domain_submit(conn, %{"site" => %{"domain" => new_domain}}) do
+    if Plausible.v2?() do
+      case Plausible.Site.Domain.change(conn.assigns.site, new_domain) do
+        {:ok, updated_site} ->
+          conn
+          |> put_flash(:success, "Website domain changed successfully")
+          |> redirect(
+            to: Routes.site_path(conn, :add_snippet_after_domain_change, updated_site.domain)
+          )
+
+        {:error, changeset} ->
+          render(conn, "change_domain.html",
+            changeset: changeset,
+            layout: {PlausibleWeb.LayoutView, "focus.html"}
+          )
+      end
+    else
+      render_error(conn, 404)
+    end
+  end
+
+  def add_snippet_after_domain_change(conn, _params) do
+    site = conn.assigns[:site] |> Repo.preload(:custom_domain)
+
+    conn
+    |> assign(:skip_plausible_tracking, true)
+    |> render("snippet_after_domain_change.html",
+      site: site,
+      layout: {PlausibleWeb.LayoutView, "focus.html"}
+    )
+  end
 end

lib/plausible_web/plugs/authorize_stats_api.ex

@@ -52,7 +52,10 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug do
   defp verify_access(_api_key, nil), do: {:error, :missing_site_id}
 
   defp verify_access(api_key, site_id) do
-    case Repo.get_by(Plausible.Site, domain: site_id) do
+    domain_based_search =
+      from s in Plausible.Site, where: s.domain == ^site_id or s.domain_changed_from == ^site_id
+
+    case Repo.one(domain_based_search) do
       %Plausible.Site{} = site ->
         is_member? = Sites.is_member?(api_key.user_id, site)
         is_super_admin? = Plausible.Auth.is_super_admin?(api_key.user_id)