Skip to content

Fix: time series trimming #5705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
macobo merged 3 commits into from
Sep 8, 2025
Merged

Fix: time series trimming #5705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b79e685
Fix time series trimming
macobo Sep 8, 2025
a5482e4
Add testing tools, test timeseries better
macobo Sep 8, 2025
2d13d1c
Minor cleanup
macobo Sep 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions lib/plausible/stats/filters/query_parser.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
def default_include(), do: @default_include

def parse(site, schema_type, params, now \\ nil) when is_map(params) do
{now, date} =
if now do
{now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()}
else
{DateTime.utc_now(:second), today(site)}
end
now = now || Plausible.Stats.Query.Test.get_fixed_now()
date = now |> DateTime.shift_zone!(site.timezone) |> DateTime.to_date()

with :ok <- JSONSchema.validate(schema_type, params),
{:ok, date, now} <- parse_date(site, Map.get(params, "date"), date, now),
Expand Down Expand Up @@ -289,8 +285,6 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end

defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()

defp parse_dimensions(dimensions) when is_list(dimensions) do
parse_list(
dimensions,
Expand Down
2 changes: 1 addition & 1 deletion lib/plausible/stats/legacy/legacy_query_builder.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange}

def from(site, params, debug_metadata, now \\ nil) do
now = now || DateTime.utc_now(:second)
now = now || Plausible.Stats.Query.Test.get_fixed_now()

query =
Query
Expand Down
15 changes: 15 additions & 0 deletions lib/plausible/stats/query/test.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
defmodule Plausible.Stats.Query.Test do
@moduledoc """
Module used in tests to 'set' the current time.
"""

@now_key :__now

def fix_now(now) do
Process.put(@now_key, now)
end

def get_fixed_now() do
Process.get(@now_key) || DateTime.utc_now(:second)
end
end
3 changes: 2 additions & 1 deletion lib/plausible/stats/timeseries.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ defmodule Plausible.Stats.Timeseries do

use Plausible
use Plausible.ClickhouseRepo
alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time}
alias Plausible.Stats.{Comparisons, Query, QueryRunner, Metrics, Time, QueryOptimizer}

@time_dimension %{
"month" => "time:month",
Expand All @@ -26,6 +26,7 @@ defmodule Plausible.Stats.Timeseries do
order_by: [{time_dimension(query), :asc}],
remove_unavailable_revenue_metrics: true
)
|> QueryOptimizer.optimize()

comparison_query =
if(query.include.comparisons,
Expand Down
3 changes: 2 additions & 1 deletion test/plausible_web/controllers/api/external_stats_controller/query_comparisons_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,12 +185,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryComparisonsTest do
build(:pageview, timestamp: ~N[2022-07-01 00:00:00])
])

Plausible.Stats.Query.Test.fix_now(~U[2022-07-01 14:00:00Z])

conn =
post(conn, "/api/v2/query-internal-test", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "91d",
"date" => "2022-07-01",
"dimensions" => ["time:day"],
"include" => %{
"time_labels" => true,
Expand Down
187 changes: 187 additions & 0 deletions test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1340,6 +1340,95 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do

assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1
end

test "trims hourly relative date range", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),
build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),
build(:pageview, timestamp: ~N[2021-01-08 23:59:00])
])

Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])

conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour"
)

assert_matches %{
"labels" => [
"2021-01-08 00:00:00",
"2021-01-08 01:00:00",
"2021-01-08 02:00:00",
"2021-01-08 03:00:00",
"2021-01-08 04:00:00",
"2021-01-08 05:00:00",
"2021-01-08 06:00:00",
"2021-01-08 07:00:00",
"2021-01-08 08:00:00"
],
"plot" => [1, 0, 0, 0, 0, 0, 1, 0, 1]
} = json_response(conn, 200)
end

test "trims monthly relative date range", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-07 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
])

Plausible.Stats.Query.Test.fix_now(~U[2021-01-07 12:00:00Z])

conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&date=2021-01-07&interval=day"
)

assert_matches %{
"labels" => [
"2021-01-01",
"2021-01-02",
"2021-01-03",
"2021-01-04",
"2021-01-05",
"2021-01-06",
"2021-01-07"
],
"plot" => [1, 0, 0, 0, 1, 0, 1]
} = json_response(conn, 200)
end

test "trims yearly relative date range", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-05 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-30 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),
build(:pageview, timestamp: ~N[2021-02-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-02-09 00:00:00])
])

Plausible.Stats.Query.Test.fix_now(~U[2021-02-07 12:00:00Z])

conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=year&metric=visitors&date=2021-02-07&interval=month"
)

assert_matches %{
"labels" => [
"2021-01-01",
"2021-02-01"
],
"plot" => [4, 1]
} = json_response(conn, 200)
end
end

describe "GET /api/stats/main-graph - comparisons" do
Expand Down Expand Up @@ -1513,6 +1602,104 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
assert last_week_plot == [33.33, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
end

test "does not trim hourly relative date range when comparing", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-08 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-08 06:05:00]),
build(:pageview, timestamp: ~N[2021-01-08 08:59:00]),
build(:pageview, timestamp: ~N[2021-01-08 23:59:00])
])

Plausible.Stats.Query.Test.fix_now(~U[2021-01-08 08:05:00Z])

conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2021-01-08&interval=hour&comparison=previous_period"
)

assert_matches %{
"labels" => [
"2021-01-08 00:00:00",
"2021-01-08 01:00:00",
"2021-01-08 02:00:00",
"2021-01-08 03:00:00",
"2021-01-08 04:00:00",
"2021-01-08 05:00:00",
"2021-01-08 06:00:00",
"2021-01-08 07:00:00",
"2021-01-08 08:00:00",
"2021-01-08 09:00:00",
"2021-01-08 10:00:00",
"2021-01-08 11:00:00",
"2021-01-08 12:00:00",
"2021-01-08 13:00:00",
"2021-01-08 14:00:00",
"2021-01-08 15:00:00",
"2021-01-08 16:00:00",
"2021-01-08 17:00:00",
"2021-01-08 18:00:00",
"2021-01-08 19:00:00",
"2021-01-08 20:00:00",
"2021-01-08 21:00:00",
"2021-01-08 22:00:00",
"2021-01-08 23:00:00"
],
"plot" => [
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1
],
"comparison_plot" => [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
} = json_response(conn, 200)
end
end

describe "GET /api/stats/main-graph - total_revenue plot" do
Expand Down
Loading