diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 255e08a7e5d8..a3116800dd2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ Make sure Docker, Elixir, Erlang and Node.js are all installed on your developme ### Start the environment -1. Run both `make postgres` and `make clickhouse`. +1. Run `make network`, `make postgres`, `make clickhouse`, `make ch-haproxy` 2. You can set up everything with `make install`, alternatively run each command separately: 1. Run `mix deps.get`. This will download the required Elixir dependencies. 2. Run `mix ecto.create`. This will create the required databases in both Postgres and Clickhouse. diff --git a/Makefile b/Makefile index 0b148dbda248..56ef834358ce 100644 --- a/Makefile +++ b/Makefile @@ -21,10 +21,16 @@ install: ## Run the initial setup server: ## Start the web server mix phx.server -CH_FLAGS ?= --detach -p 8123:8123 -p 9000:9000 --ulimit nofile=262144:262144 --name plausible_clickhouse --env CLICKHOUSE_SKIP_USER_SETUP=1 +CH_FLAGS ?= --detach -p 8123:8123 --ulimit nofile=262144:262144 --name plausible_clickhouse --env CLICKHOUSE_SKIP_USER_SETUP=1 + +network: ## Create a docker network for clickhouse and haproxy + docker network create ch-net || true clickhouse: ## Start a container with a recent version of clickhouse - docker run $(CH_FLAGS) --network host --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:latest-alpine + docker run $(CH_FLAGS) --network ch-net --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:latest-alpine + +ch-haproxy: ## Start a container with a recent version of clickhouse + docker run --detach -p 8124:8123 --name haproxy --network ch-net --volume=$$PWD/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro haproxy:2.9 clickhouse-client: ## Connect to clickhouse docker exec -it plausible_clickhouse clickhouse-client -d plausible_events_db diff --git a/config/.env.dev b/config/.env.dev index c1b5361dcecc..09a83a731f57 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -1,7 +1,7 @@ BASE_URL=http://localhost:8000 SECURE_COOKIE=false DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_dev -CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_events_db +CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8124/plausible_events_db CLICKHOUSE_MAX_BUFFER_SIZE_BYTES=1000000 SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg TOTP_VAULT_KEY=Q3BD4nddbkVJIPXgHuo5NthGKSIH0yesRfG05J88HIo= diff --git a/config/.env.test b/config/.env.test index 42a1e8552634..dba6d320cfcd 100644 --- a/config/.env.test +++ b/config/.env.test @@ -1,5 +1,5 @@ DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_test -CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_test +CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8124/plausible_test SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg TOTP_VAULT_KEY=1Jah1HEOnCEnmBE+4/OgbJRraJIppPmYCNbZoFJboZs= BASE_URL=http://localhost:8000 diff --git a/config/runtime.exs b/config/runtime.exs index 0d2e62860151..6fb6a3bb9242 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -114,7 +114,7 @@ ch_db_url = get_var_from_path_or_env( config_dir, "CLICKHOUSE_DATABASE_URL", - "http://plausible_events_db:8123/plausible_events_db" + "http://plausible_events_db:8124/plausible_events_db" ) {ingest_pool_size, ""} = @@ -634,7 +634,7 @@ config :plausible, Plausible.ClickhouseRepo, # NB! when :timeout is overridden to be over 20s, # for it to have meaningful effect, # this must be overridden as well - max_execution_time: 20 + # max_execution_time: 20 ] config :plausible, Plausible.IngestRepo, diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 000000000000..1dfaff28b88d --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,36 @@ +global + daemon + maxconn 20000 + # the default of 16384 bytes was too small + # clickhouse http_max_uri_size is 104857600 + tune.bufsize 104857600 + +defaults + # mode is inherited by sections that follow + mode http + option dontlognull + option httplog + option dontlog-normal + option abortonclose + timeout connect 10s + timeout client 100s + timeout server 100s + log global + +frontend clickhouse_http + mode http + + # receives traffic from clients + bind *:8123 + + default_backend clickhouse_http + + log /var/log/haproxy.log local0 + +backend clickhouse_http + mode http + balance roundrobin + + option httpchk GET /ping + http-check expect string Ok. + server ch1 plausible_clickhouse:8123 check diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index cf5c0dedf764..97cc3016fcc7 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -59,13 +59,45 @@ defmodule Plausible.ClickhouseRepo do log_comment = Jason.encode!(log_comment_data) opts = - Keyword.update(opts, :settings, [log_comment: log_comment], fn settings -> - [{:log_comment, log_comment} | settings] + opts + |> Keyword.update(:settings, [log_comment: log_comment], fn current_settings -> + [{:log_comment, log_comment} | current_settings] end) + opts = + if plausible_query do + opts + |> Keyword.update!(:settings, fn current_settings -> + Enum.concat(get_extra_connection_settings(log_comment_data), current_settings) + end) + else + opts + end + {query, opts} end + defp get_extra_connection_settings(%{params: params}) do + keys = + params + |> Map.keys() + |> Enum.filter(fn k -> + case k do + "clickhouse_readonly" -> false + "clickhouse_" <> _k -> true + _ -> false + end + end) + + Enum.map(keys, fn k -> + {String.to_atom(String.trim_leading(k, "clickhouse_")), params[k]} + end) + end + + defp get_extra_connection_settings(_) do + [] + end + def get_config_without_ch_query_execution_timeout() do {settings, config} = Plausible.ClickhouseRepo.config() |> Keyword.pop!(:settings) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index ed3b7f930fe6..bd57fd366270 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -1522,8 +1522,9 @@ defmodule PlausibleWeb.Api.StatsController do end def current_visitors(conn, _) do - site = conn.assigns[:site] - json(conn, Stats.current_visitors(site)) + # site = conn.assigns[:site] + Plausible.ClickhouseRepo.query("SELECT 'direct_test', count(*) FROM numbers(8000000000) WHERE sipHash64(number) % 1000000 = 0") + json(conn, %{"finish" => true}) end defp google_api(), do: Application.fetch_env!(:plausible, :google_api)