Skip to content

Limits (WIP) #6032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
apata wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Limits (WIP) #6032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73d6b82
WIP
apata Jan 28, 2026
254a756
Local HAProxy - mode HTTP works with cancel_http_readonly_queries_on_…
apata Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Make sure Docker, Elixir, Erlang and Node.js are all installed on your developme

### Start the environment

1. Run both `make postgres` and `make clickhouse`.
1. Run `make network`, `make postgres`, `make clickhouse`, `make ch-haproxy`
2. You can set up everything with `make install`, alternatively run each command separately:
1. Run `mix deps.get`. This will download the required Elixir dependencies.
2. Run `mix ecto.create`. This will create the required databases in both Postgres and Clickhouse.
Expand Down
10 changes: 8 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,16 @@ install: ## Run the initial setup
server: ## Start the web server
mix phx.server

CH_FLAGS ?= --detach -p 8123:8123 -p 9000:9000 --ulimit nofile=262144:262144 --name plausible_clickhouse --env CLICKHOUSE_SKIP_USER_SETUP=1
CH_FLAGS ?= --detach -p 8123:8123 --ulimit nofile=262144:262144 --name plausible_clickhouse --env CLICKHOUSE_SKIP_USER_SETUP=1

network: ## Create a docker network for clickhouse and haproxy
docker network create ch-net || true

clickhouse: ## Start a container with a recent version of clickhouse
docker run $(CH_FLAGS) --network host --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:latest-alpine
docker run $(CH_FLAGS) --network ch-net --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:latest-alpine

ch-haproxy: ## Start a container with a recent version of clickhouse
docker run --detach -p 8124:8123 --name haproxy --network ch-net --volume=$$PWD/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro haproxy:2.9

clickhouse-client: ## Connect to clickhouse
docker exec -it plausible_clickhouse clickhouse-client -d plausible_events_db
Expand Down
2 changes: 1 addition & 1 deletion config/.env.dev
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
BASE_URL=http://localhost:8000
SECURE_COOKIE=false
DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_dev
CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_events_db
CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8124/plausible_events_db
CLICKHOUSE_MAX_BUFFER_SIZE_BYTES=1000000
SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg
TOTP_VAULT_KEY=Q3BD4nddbkVJIPXgHuo5NthGKSIH0yesRfG05J88HIo=
Expand Down
2 changes: 1 addition & 1 deletion config/.env.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_test
CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_test
CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8124/plausible_test
SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg
TOTP_VAULT_KEY=1Jah1HEOnCEnmBE+4/OgbJRraJIppPmYCNbZoFJboZs=
BASE_URL=http://localhost:8000
Expand Down
4 changes: 2 additions & 2 deletions config/runtime.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ ch_db_url =
get_var_from_path_or_env(
config_dir,
"CLICKHOUSE_DATABASE_URL",
"http://plausible_events_db:8123/plausible_events_db"
"http://plausible_events_db:8124/plausible_events_db"
)

{ingest_pool_size, ""} =
Expand Down Expand Up @@ -634,7 +634,7 @@ config :plausible, Plausible.ClickhouseRepo,
# NB! when :timeout is overridden to be over 20s,
# for it to have meaningful effect,
# this must be overridden as well
max_execution_time: 20
# max_execution_time: 20
]

config :plausible, Plausible.IngestRepo,
Expand Down
36 changes: 36 additions & 0 deletions haproxy.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
global
daemon
maxconn 20000
# the default of 16384 bytes was too small
# clickhouse http_max_uri_size is 104857600
tune.bufsize 104857600

defaults
# mode is inherited by sections that follow
mode http
option dontlognull
option httplog
option dontlog-normal
option abortonclose
timeout connect 10s
timeout client 100s
timeout server 100s
log global

frontend clickhouse_http
mode http

# receives traffic from clients
bind *:8123

default_backend clickhouse_http

log /var/log/haproxy.log local0

backend clickhouse_http
mode http
balance roundrobin

option httpchk GET /ping
http-check expect string Ok.
server ch1 plausible_clickhouse:8123 check
36 changes: 34 additions & 2 deletions lib/plausible/clickhouse_repo.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,45 @@ defmodule Plausible.ClickhouseRepo do
log_comment = Jason.encode!(log_comment_data)

opts =
Keyword.update(opts, :settings, [log_comment: log_comment], fn settings ->
[{:log_comment, log_comment} | settings]
opts
|> Keyword.update(:settings, [log_comment: log_comment], fn current_settings ->
[{:log_comment, log_comment} | current_settings]
end)

opts =
if plausible_query do
opts
|> Keyword.update!(:settings, fn current_settings ->
Enum.concat(get_extra_connection_settings(log_comment_data), current_settings)
end)
else
opts
end

{query, opts}
end

defp get_extra_connection_settings(%{params: params}) do
keys =
params
|> Map.keys()
|> Enum.filter(fn k ->
case k do
"clickhouse_readonly" -> false
"clickhouse_" <> _k -> true
_ -> false
end
end)

Enum.map(keys, fn k ->
{String.to_atom(String.trim_leading(k, "clickhouse_")), params[k]}
end)
end

defp get_extra_connection_settings(_) do
[]
end

def get_config_without_ch_query_execution_timeout() do
{settings, config} = Plausible.ClickhouseRepo.config() |> Keyword.pop!(:settings)

Expand Down
5 changes: 3 additions & 2 deletions lib/plausible_web/controllers/api/stats_controller.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1522,8 +1522,9 @@ defmodule PlausibleWeb.Api.StatsController do
end

def current_visitors(conn, _) do
site = conn.assigns[:site]
json(conn, Stats.current_visitors(site))
# site = conn.assigns[:site]
Plausible.ClickhouseRepo.query("SELECT 'direct_test', count(*) FROM numbers(8000000000) WHERE sipHash64(number) % 1000000 = 0")
json(conn, %{"finish" => true})
end

defp google_api(), do: Application.fetch_env!(:plausible, :google_api)
Expand Down
Loading