diff --git a/config/config.exs b/config/config.exs index 7aab559e858a..f25fa5313a21 100644 --- a/config/config.exs +++ b/config/config.exs @@ -96,7 +96,12 @@ config :plausible, cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) -crontab = [ +base_cron = [ + # Daily at midnight + {"0 0 * * *", Plausible.Workers.RotateSalts}, +] + +extra_cron = [ # hourly {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, #  hourly @@ -111,7 +116,8 @@ crontab = [ {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates} ] -queues = [ +base_queues = [rotate_salts: 1] +extra_queues = [ provision_ssl_certificates: 1, fetch_tweets: 1, check_stats_emails: 1, @@ -123,8 +129,8 @@ queues = [ config :plausible, Oban, repo: Plausible.Repo, - queues: if(cron_enabled, do: queues, else: []), - crontab: if(cron_enabled, do: crontab, else: false) + queues: if(cron_enabled, do: base_queues ++ extra_queues, else: base_queues), + crontab: if(cron_enabled, do: base_cron ++ extra_cron, else: base_cron) config :plausible, :google, client_id: System.get_env("GOOGLE_CLIENT_ID"), diff --git a/config/releases.exs b/config/releases.exs index 2c571428f152..29d07158010c 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -146,7 +146,12 @@ config :plausible, :custom_domain_server, password: custom_domain_server_password, ip: custom_domain_server_ip -crontab = [ +base_cron = [ + # Daily at midnight + {"0 0 * * *", Plausible.Workers.RotateSalts}, +] + +extra_cron = [ # hourly {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, #  hourly @@ -161,7 +166,8 @@ crontab = [ {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates} ] -queues = [ +base_queues = [rotate_salts: 1] +extra_queues = [ provision_ssl_certificates: 1, fetch_tweets: 1, check_stats_emails: 1, @@ -173,8 +179,8 @@ queues = [ config :plausible, Oban, repo: Plausible.Repo, - queues: if(cron_enabled, do: queues, else: []), - crontab: if(cron_enabled, do: crontab, else: false) + queues: if(cron_enabled, do: base_queues ++ extra_queues, else: base_queues), + crontab: if(cron_enabled, do: base_cron ++ extra_cron, else: base_cron) config :ref_inspector, init: {Plausible.Release, :configure_ref_inspector} diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index bae4a9751460..199d59b7def3 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -15,6 +15,7 @@ defmodule Plausible.Application do Keyword.merge([scheme: :http, port: 8123, name: :clickhouse], clickhouse_config) ), Plausible.Session.Store, + Plausible.Session.Salts, {Oban, Application.get_env(:plausible, Oban)} ] diff --git a/lib/plausible/session/salts.ex b/lib/plausible/session/salts.ex new file mode 100644 index 000000000000..a28048c06009 --- /dev/null +++ b/lib/plausible/session/salts.ex @@ -0,0 +1,46 @@ +defmodule Plausible.Session.Salts do + use Agent + use Plausible.Repo + + def start_link(_opts) do + Agent.start_link(fn -> + clean_old_salts() + salts = Repo.all(from s in "salts", select: s.salt, order_by: [desc: s.inserted_at], limit: 2) + case salts do + [current, prev] -> + %{previous: prev, current: current} + [current] -> + %{previous: nil, current: current} + [] -> + new = generate_and_persist_new_salt() + %{previous: nil, current: new} + end + end, name: __MODULE__) + end + + def fetch() do + Agent.get(__MODULE__, & &1) + end + + def rotate() do + Agent.update(__MODULE__, fn %{current: current} -> + clean_old_salts() + + %{ + current: generate_and_persist_new_salt(), + previous: current + } + end) + end + + defp generate_and_persist_new_salt() do + salt = :crypto.strong_rand_bytes(16) + + Repo.insert_all("salts", [%{salt: salt, inserted_at: Timex.now()}]) + salt + end + + defp clean_old_salts() do + Repo.delete_all(from s in "salts", where: s.inserted_at < fragment("now() - '48 hours'::interval")) + end +end diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index 248a1b74cba5..e16703ba837f 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -6,7 +6,6 @@ defmodule Plausible.Session.Store do require Logger @session_length_seconds Application.get_env(:plausible, :session_length_minutes) * 60 - # Remember session for longer in case of upstream latency @forget_session_after @session_length_seconds * 2 @garbage_collect_interval_milliseconds 60 * 1000 @@ -43,12 +42,12 @@ defmodule Plausible.Session.Store do {:ok, %{timer: timer, sessions: sessions}} end - def on_event(event) do - GenServer.call(__MODULE__, {:on_event, event}) + def on_event(event, prev_user_id) do + GenServer.call(__MODULE__, {:on_event, event, prev_user_id}) end - def handle_call({:on_event, event}, _from, %{sessions: sessions} = state) do - found_session = sessions[event.user_id] + def handle_call({:on_event, event, prev_user_id}, _from, %{sessions: sessions} = state) do + found_session = sessions[event.user_id] || (prev_user_id && sessions[prev_user_id]) active = is_active?(found_session, event) updated_sessions = @@ -80,7 +79,8 @@ defmodule Plausible.Session.Store do defp update_session(session, event) do %{ session - | timestamp: event.timestamp, + | user_id: event.user_id, + timestamp: event.timestamp, exit_page: event.pathname, is_bounce: false, duration: Timex.diff(event.timestamp, session.start, :second), diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index 30b8ff5f58b1..c9dcc5c009e1 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -42,6 +42,7 @@ defmodule PlausibleWeb.Api.ExternalController do ref = parse_referrer(uri, params["referrer"]) country_code = visitor_country(conn) + salts = Plausible.Session.Salts.fetch() event_attrs = %{ timestamp: NaiveDateTime.utc_now(), @@ -49,7 +50,7 @@ defmodule PlausibleWeb.Api.ExternalController do hostname: strip_www(uri && uri.host), domain: strip_www(params["domain"]) || strip_www(uri && uri.host), pathname: uri && (uri.path || "/"), - user_id: generate_user_id(conn, params), + user_id: generate_user_id(conn, params, salts[:current]), country_code: country_code, operating_system: ua && os_name(ua), browser: ua && browser_name(ua), @@ -61,8 +62,9 @@ defmodule PlausibleWeb.Api.ExternalController do changeset = Plausible.ClickhouseEvent.changeset(%Plausible.ClickhouseEvent{}, event_attrs) if changeset.valid? do + previous_user_id = salts[:previous] && generate_user_id(conn, params, salts[:previous]) event = struct(Plausible.ClickhouseEvent, event_attrs) - session_id = Plausible.Session.Store.on_event(event) + session_id = Plausible.Session.Store.on_event(event, previous_user_id) Map.put(event, :session_id, session_id) |> Plausible.Event.WriteBuffer.insert() @@ -105,16 +107,12 @@ defmodule PlausibleWeb.Api.ExternalController do end end - defp generate_user_id(conn, params) do - hash_key = - Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base) - |> binary_part(0, 16) - + defp generate_user_id(conn, params, salt) do user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || "" ip_address = get_ip(conn) domain = strip_www(params["domain"]) || "" - SipHash.hash!(hash_key, user_agent <> ip_address <> domain) + SipHash.hash!(salt, user_agent <> ip_address <> domain) end defp calculate_screen_size(nil), do: nil diff --git a/lib/workers/rotate_salts.ex b/lib/workers/rotate_salts.ex new file mode 100644 index 000000000000..8cde11389206 --- /dev/null +++ b/lib/workers/rotate_salts.ex @@ -0,0 +1,9 @@ +defmodule Plausible.Workers.RotateSalts do + use Plausible.Repo + use Oban.Worker, queue: :rotate_salts + + @impl Oban.Worker + def perform(_args, _job) do + Plausible.Session.Salts.rotate() + end +end diff --git a/priv/repo/migrations/20200619071221_create_salts_table.exs b/priv/repo/migrations/20200619071221_create_salts_table.exs new file mode 100644 index 000000000000..0057bc942ed9 --- /dev/null +++ b/priv/repo/migrations/20200619071221_create_salts_table.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CreateSaltsTable do + use Ecto.Migration + + def change do + create table(:salts) do + add :salt, :bytea, null: false + + timestamps(updated_at: false) + end + end +end