diff --git a/CHANGELOG.md b/CHANGELOG.md index 6925d3355d54..179be7be0a7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ All notable changes to this project will be documented in this file. - Add Yesterday as an time range option in the dashboard - Add support for importing Google Analytics 4 data - Import custom events from Google Analytics 4 +- Ability to filter Search Console keywords by page, country and device plausible/analytics#4077 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100 ### Removed diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js index da2135d79920..f8b2f71b0ade 100644 --- a/assets/js/dashboard/stats/modals/exit-pages.js +++ b/assets/js/dashboard/stats/modals/exit-pages.js @@ -4,7 +4,7 @@ import { withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' -import numberFormatter from '../../util/number-formatter' +import numberFormatter, {percentageFormatter} from '../../util/number-formatter' import { parseQuery } from '../../query' import { trimURL } from '../../util/url' class ExitPagesModal extends React.Component { @@ -34,14 +34,6 @@ class ExitPagesModal extends React.Component { this.setState({ loading: true, page: this.state.page + 1 }, this.loadPages.bind(this)) } - formatPercentage(number) { - if (typeof (number) === 'number') { - return number + '%' - } else { - return '-' - } - } - showConversionRate() { return !!this.state.query.filters.goal } @@ -74,7 +66,7 @@ class ExitPagesModal extends React.Component { {this.showConversionRate() && {numberFormatter(page.total_visitors)}} {numberFormatter(page.visitors)} {this.showExtra() && {numberFormatter(page.visits)}} - {this.showExtra() && {this.formatPercentage(page.exit_rate)}} + {this.showExtra() && {percentageFormatter(page.exit_rate)}} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%} ) diff --git a/assets/js/dashboard/stats/modals/google-keywords.js b/assets/js/dashboard/stats/modals/google-keywords.js index 3856ccd65127..39ba7f3b8fca 100644 --- a/assets/js/dashboard/stats/modals/google-keywords.js +++ b/assets/js/dashboard/stats/modals/google-keywords.js @@ -3,7 +3,7 @@ import { Link, withRouter } from 'react-router-dom' import Modal from './modal' import * as api from '../../api' -import numberFormatter from '../../util/number-formatter' +import numberFormatter, { percentageFormatter } from '../../util/number-formatter' import {parseQuery} from '../../query' import RocketIcon from './rocket-icon' @@ -17,49 +17,31 @@ class GoogleKeywordsModal extends React.Component { } componentDidMount() { - if (this.state.query.filters.goal) { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers/Google`, this.state.query, {limit: 100}) - .then((res) => this.setState({ - loading: false, - searchTerms: res.search_terms, - totalVisitors: res.total_visitors, - notConfigured: res.not_configured, - isOwner: res.is_owner - })) - } else { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100}) - .then((res) => this.setState({ - loading: false, - searchTerms: res.search_terms, - totalVisitors: res.total_visitors, - notConfigured: res.not_configured, - isOwner: res.is_owner - })) - } + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100}) + .then((res) => this.setState({ + loading: false, + searchTerms: res.search_terms, + notConfigured: res.not_configured, + isOwner: res.is_owner + })) } renderTerm(term) { return ( - - {term.name} + {term.name} {numberFormatter(term.visitors)} + {numberFormatter(term.impressions)} + {percentageFormatter(term.ctr)} + {numberFormatter(term.position)} ) } renderKeywords() { - if (this.state.query.filters.goal) { - return ( -
- -
Sorry, we cannot show which keywords converted best for goal {this.state.query.filters.goal}
-
Google does not share this information
-
- ) - } else if (this.state.notConfigured) { + if (this.state.notConfigured) { if (this.state.isOwner) { return (
@@ -84,7 +66,10 @@ class GoogleKeywordsModal extends React.Component { Search Term - Visitors + Visitors + Impressions + CTR + Position @@ -102,14 +87,6 @@ class GoogleKeywordsModal extends React.Component { } } - renderGoalText() { - if (this.state.query.filters.goal) { - return ( -

completed {this.state.query.filters.goal}

- ) - } - } - renderBody() { if (this.state.loading) { return ( @@ -122,10 +99,6 @@ class GoogleKeywordsModal extends React.Component {
-

- {this.state.totalVisitors} visitors from Google
-

- {this.renderGoalText()} { this.renderKeywords() }
diff --git a/assets/js/dashboard/stats/sources/search-terms.js b/assets/js/dashboard/stats/sources/search-terms.js index 14863c67d25b..2046e3db5932 100644 --- a/assets/js/dashboard/stats/sources/search-terms.js +++ b/assets/js/dashboard/stats/sources/search-terms.js @@ -39,7 +39,8 @@ export default class SearchTerms extends React.Component { loading: false, searchTerms: res.search_terms || [], notConfigured: res.not_configured, - isAdmin: res.is_admin + isAdmin: res.is_admin, + unsupportedFilters: res.unsupported_filters })).catch((error) => { this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin }) @@ -68,21 +69,19 @@ export default class SearchTerms extends React.Component { } renderList() { - if (this.props.query.filters.goal) { + if (this.state.unsupportedFilters) { return (
-
Sorry, we cannot show which keywords converted best for goal {this.props.query.filters.goal}
-
Google does not share this information
+
Unable to fetch keyword data from Search Console because it does not support the current set of filters
) - } else if (this.state.notConfigured) { return (
- This site is not connected to Search Console so we cannot show the search phrases. + This site is not connected to Search Console so we cannot show the search terms {this.state.isAdmin && this.state.error && <>

Please click below to connect your Search Console account.

}
{this.state.isAdmin && Connect with Google } @@ -103,9 +102,8 @@ export default class SearchTerms extends React.Component { ) } else { return ( -
- -
No search terms were found for this period. Please adjust or extend your time range. Check our documentation for more details.
+
+
No data yet
) } diff --git a/assets/js/dashboard/util/number-formatter.js b/assets/js/dashboard/util/number-formatter.js index c724525ef8a8..48ced1e436d0 100644 --- a/assets/js/dashboard/util/number-formatter.js +++ b/assets/js/dashboard/util/number-formatter.js @@ -49,3 +49,11 @@ export function durationFormatter(duration) { return `${seconds}s` } } + +export function percentageFormatter(number) { + if (typeof (number) === 'number') { + return number + '%' + } else { + return '-' + } +} diff --git a/fixture/http_mocks/google_analytics_stats#without_page.json b/fixture/http_mocks/google_analytics_stats#without_page.json deleted file mode 100644 index 546cc140de5a..000000000000 --- a/fixture/http_mocks/google_analytics_stats#without_page.json +++ /dev/null @@ -1,41 +0,0 @@ -[ - { - "status": 200, - "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query", - "method": "post", - "request_body": { - "dimensionFilterGroups": {}, - "dimensions": [ - "query" - ], - "endDate": "2022-01-05", - "rowLimit": 5, - "startDate": "2022-01-01" - }, - "response_body": { - "responseAggregationType": "auto", - "rows": [ - { - "clicks": 25.0, - "ctr": 0.3, - "impressions": 50.0, - "keys": [ - "keyword1", - "keyword2" - ], - "position": 2.0 - }, - { - "clicks": 15.0, - "ctr": 0.5, - "impressions": 25.0, - "keys": [ - "keyword3", - "keyword4" - ], - "position": 4.0 - } - ] - } - } -] \ No newline at end of file diff --git a/fixture/http_mocks/google_analytics_stats.json b/fixture/http_mocks/google_analytics_stats.json index 546cc140de5a..431b8fa2a7d3 100644 --- a/fixture/http_mocks/google_analytics_stats.json +++ b/fixture/http_mocks/google_analytics_stats.json @@ -4,7 +4,7 @@ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query", "method": "post", "request_body": { - "dimensionFilterGroups": {}, + "dimensionFilterGroups": [], "dimensions": [ "query" ], @@ -16,26 +16,20 @@ "responseAggregationType": "auto", "rows": [ { - "clicks": 25.0, - "ctr": 0.3, - "impressions": 50.0, - "keys": [ - "keyword1", - "keyword2" - ], - "position": 2.0 + "clicks": 25, + "ctr": 0.3679, + "impressions": 50, + "keys": ["keyword1"], + "position": 2.2312312 }, { - "clicks": 15.0, + "clicks": 15, "ctr": 0.5, - "impressions": 25.0, - "keys": [ - "keyword3", - "keyword4" - ], + "impressions": 25, + "keys": ["keyword3"], "position": 4.0 } ] } } -] \ No newline at end of file +] diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 3e863c2f821a..3fd63c87f760 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -6,6 +6,7 @@ defmodule Plausible.Google.API do use Timex alias Plausible.Google.HTTP + alias Plausible.Google.SearchConsole require Logger @@ -74,21 +75,26 @@ defmodule Plausible.Google.API do end def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do - with site <- Plausible.Repo.preload(site, :google_auth), + with {:ok, site} <- ensure_search_console_property(site), {:ok, access_token} <- maybe_refresh_token(site.google_auth), + {:ok, search_console_filters} <- + SearchConsole.Filters.transform(site.google_auth.property, filters), {:ok, stats} <- HTTP.list_stats( access_token, site.google_auth.property, date_range, limit, - filters["page"] + search_console_filters ) do stats |> Map.get("rows", []) - |> Enum.filter(fn row -> row["clicks"] > 0 end) - |> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end) + |> Enum.map(&search_console_row/1) |> then(&{:ok, &1}) + else + :google_property_not_configured -> {:error, :google_property_not_configured} + :unsupported_filters -> {:error, :unsupported_filters} + {:error, error} -> {:error, error} end end @@ -142,6 +148,44 @@ defmodule Plausible.Google.API do Timex.before?(expires_at, thirty_seconds_ago) end + defp ensure_search_console_property(site) do + site = Plausible.Repo.preload(site, :google_auth) + + if site.google_auth && site.google_auth.property do + {:ok, site} + else + :google_property_not_configured + end + end + + defp search_console_row(row) do + %{ + # We always request just one dimension at a time (`query`) + name: row["keys"] |> List.first(), + visitors: row["clicks"], + impressions: row["impressions"], + ctr: rounded_ctr(row["ctr"]), + position: rounded_position(row["position"]) + } + end + + defp rounded_ctr(ctr) do + {:ok, decimal} = Decimal.cast(ctr) + + decimal + |> Decimal.mult(100) + |> Decimal.round(1) + |> Decimal.to_float() + end + + defp rounded_position(position) do + {:ok, decimal} = Decimal.cast(position) + + decimal + |> Decimal.round(1) + |> Decimal.to_float() + end + defp client_id() do Keyword.fetch!(Application.get_env(:plausible, :google), :client_id) end diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex index be55fbcf5e54..c24808764bb1 100644 --- a/lib/plausible/google/http.ex +++ b/lib/plausible/google/http.ex @@ -39,26 +39,18 @@ defmodule Plausible.Google.HTTP do response.body end - def list_stats(access_token, property, date_range, limit, page \\ nil) do - property = URI.encode_www_form(property) - - filter_groups = - if page do - url = property_base_url(property) - [%{filters: [%{dimension: "page", expression: "https://#{url}#{page}"}]}] - else - %{} - end - + def list_stats(access_token, property, date_range, limit, search_console_filters) do params = %{ startDate: Date.to_iso8601(date_range.first), endDate: Date.to_iso8601(date_range.last), dimensions: ["query"], rowLimit: limit, - dimensionFilterGroups: filter_groups + dimensionFilterGroups: search_console_filters } - url = "#{api_url()}/webmasters/v3/sites/#{property}/searchAnalytics/query" + url = + "#{api_url()}/webmasters/v3/sites/#{URI.encode_www_form(property)}/searchAnalytics/query" + headers = [{"Authorization", "Bearer #{access_token}"}] case HTTPClient.impl().post(url, headers, params) do @@ -78,9 +70,6 @@ defmodule Plausible.Google.HTTP do end end - defp property_base_url("sc-domain:" <> domain), do: "https://" <> domain - defp property_base_url(url), do: url - def refresh_auth_token(refresh_token) do url = "#{api_url()}/oauth2/v4/token" headers = [{"content-type", "application/x-www-form-urlencoded"}] diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex new file mode 100644 index 000000000000..4e0e00d87572 --- /dev/null +++ b/lib/plausible/google/search_console/filters.ex @@ -0,0 +1,83 @@ +defmodule Plausible.Google.SearchConsole.Filters do + @moduledoc false + import Plausible.Stats.Base, only: [page_regex: 1] + + def transform(property, plausible_filters) do + plausible_filters = Map.drop(plausible_filters, ["visit:source"]) + + search_console_filters = + Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters -> + case transform_filter(property, plausible_filter) do + :unsupported -> {:halt, :unsupported_filters} + search_console_filter -> {:cont, [search_console_filter | search_console_filters]} + end + end) + + case search_console_filters do + :unsupported_filters -> :unsupported_filters + [] -> {:ok, []} + filters when is_list(filters) -> {:ok, [%{filters: filters}]} + end + end + + defp transform_filter(property, {"event:page", filter}) do + transform_filter(property, {"visit:entry_page", filter}) + end + + defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do + %{dimension: "page", expression: property_url(property, page)} + end + + defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do + expression = + Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end) + + %{dimension: "page", operator: "includingRegex", expression: expression} + end + + defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do + page = page_regex(property_url(property, page)) + %{dimension: "page", operator: "includingRegex", expression: page} + end + + defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}}) + when is_list(pages) do + expression = + Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end) + + %{dimension: "page", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do + %{dimension: "device", expression: search_console_device(device)} + end + + defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do + expression = devices |> Enum.join("|") + %{dimension: "device", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do + %{dimension: "country", expression: search_console_country(country)} + end + + defp transform_filter(_property, {"visit:country", {:member, countries}}) + when is_list(countries) do + expression = Enum.map_join(countries, "|", &search_console_country/1) + %{dimension: "country", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_, _filter), do: :unsupported + + defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page + defp property_url(url, page), do: url <> page + + defp search_console_device("Desktop"), do: "DESKTOP" + defp search_console_device("Mobile"), do: "MOBILE" + defp search_console_device("Tablet"), do: "TABLET" + + defp search_console_country(alpha_2) do + country = Location.Country.get_country(alpha_2) + country.alpha_3 + end +end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 66659f3e5d17..dc2a562cdd08 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -680,36 +680,29 @@ defmodule PlausibleWeb.Api.StatsController do end def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do - site = conn.assigns[:site] |> Repo.preload(:google_auth) - - query = - Query.from(site, params) - |> Query.put_filter("visit:source", "Google") - - search_terms = - if site.google_auth && site.google_auth.property && !query.filters["goal"] do - google_api().fetch_stats(site, query, params["limit"] || 9) - end + site = conn.assigns[:site] - %{:visitors => %{value: total_visitors}} = Stats.aggregate(site, query, [:visitors]) + query = Query.from(site, params) user_id = get_session(conn, :current_user_id) is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site) - case search_terms do - nil -> - json(conn, %{not_configured: true, is_admin: is_admin, total_visitors: total_visitors}) + case google_api().fetch_stats(site, query, params["limit"] || 9) do + {:error, :google_propery_not_configured} -> + json(conn, %{not_configured: true, is_admin: is_admin}) + + {:error, :unsupported_filters} -> + json(conn, %{unsupported_filters: true}) {:ok, terms} -> - json(conn, %{search_terms: terms, total_visitors: total_visitors}) + json(conn, %{search_terms: terms}) {:error, _} -> conn |> put_status(502) |> json(%{ not_configured: true, - is_admin: is_admin, - total_visitors: total_visitors + is_admin: is_admin }) end end diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs index 99bc350c6d94..eca4fab00e37 100644 --- a/test/plausible/google/api_test.exs +++ b/test/plausible/google/api_test.exs @@ -245,7 +245,7 @@ defmodule Plausible.Google.APITest do "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query", [{"Authorization", "Bearer 123"}], %{ - dimensionFilterGroups: %{}, + dimensionFilterGroups: [], dimensions: ["query"], endDate: "2022-01-05", rowLimit: 5, @@ -301,29 +301,31 @@ defmodule Plausible.Google.APITest do end end - describe "fetch_stats/3 with VCR cassetes" do - test "returns name and visitor count", %{user: user, site: site} do - mock_http_with("google_analytics_stats.json") + test "returns error when token refresh fails", %{user: user, site: site} do + mock_http_with("google_analytics_auth#invalid_grant.json") - insert(:google_auth, - user: user, - site: site, - property: "sc-domain:dummy.test", - expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600) - ) + insert(:google_auth, + user: user, + site: site, + property: "sc-domain:dummy.test", + access_token: "*****", + refresh_token: "*****", + expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600) + ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} - assert {:ok, - [ - %{name: ["keyword1", "keyword2"], visitors: 25}, - %{name: ["keyword3", "keyword4"], visitors: 15} - ]} = Google.API.fetch_stats(site, query, 5) - end + assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5) + end + + test "returns error when google auth not configured", %{site: site} do + query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} - test "returns next page when page argument is set", %{user: user, site: site} do - mock_http_with("google_analytics_stats#with_page.json") + assert {:error, :google_property_not_configured} = Google.API.fetch_stats(site, query, 5) + end + describe "fetch_stats/3 with valid auth" do + setup %{user: user, site: site} do insert(:google_auth, user: user, site: site, @@ -331,52 +333,64 @@ defmodule Plausible.Google.APITest do expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600) ) - query = %Plausible.Stats.Query{ - filters: %{"page" => 5}, - date_range: Date.range(~D[2022-01-01], ~D[2022-01-05]) - } - - assert {:ok, - [ - %{name: ["keyword1", "keyword2"], visitors: 25}, - %{name: ["keyword3", "keyword4"], visitors: 15} - ]} = Google.API.fetch_stats(site, query, 5) + :ok end - test "defaults first page when page argument is not set", %{user: user, site: site} do - mock_http_with("google_analytics_stats#without_page.json") - - insert(:google_auth, - user: user, - site: site, - property: "sc-domain:dummy.test", - expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600) - ) + test "returns name and visitor count", %{site: site} do + mock_http_with("google_analytics_stats.json") query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} assert {:ok, [ - %{name: ["keyword1", "keyword2"], visitors: 25}, - %{name: ["keyword3", "keyword4"], visitors: 15} + %{name: "keyword1", visitors: 25, ctr: 36.8, impressions: 50, position: 2.2}, + %{name: "keyword3", visitors: 15} ]} = Google.API.fetch_stats(site, query, 5) end - test "returns error when token refresh fails", %{user: user, site: site} do - mock_http_with("google_analytics_auth#invalid_grant.json") - - insert(:google_auth, - user: user, - site: site, - property: "sc-domain:dummy.test", - access_token: "*****", - refresh_token: "*****", - expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600) + test "transforms page filters to search console format", %{site: site} do + expect( + Plausible.HTTPClient.Mock, + :post, + fn + "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query", + [{"Authorization", "Bearer 123"}], + %{ + dimensionFilterGroups: [ + %{filters: [%{expression: "https://dummy.test/page", dimension: "page"}]} + ], + dimensions: ["query"], + endDate: "2022-01-05", + rowLimit: 5, + startDate: "2022-01-01" + } -> + {:ok, %Finch.Response{status: 200, body: %{"rows" => []}}} + end ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Plausible.Stats.Query.from(site, %{ + "period" => "custom", + "from" => "2022-01-01", + "to" => "2022-01-05", + "filters" => "event:page==/page" + }) + + assert {:ok, []} = Google.API.fetch_stats(site, query, 5) + end - assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5) + test "returns :invalid filters when using filters that cannot be used in Search Console", %{ + site: site + } do + query = + Plausible.Stats.Query.from(site, %{ + "period" => "custom", + "from" => "2022-01-01", + "to" => "2022-01-05", + "filters" => "event:goal==Signup" + }) + + assert {:error, :unsupported_filters} = Google.API.fetch_stats(site, query, 5) end end diff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exs new file mode 100644 index 000000000000..7d73dacda8b9 --- /dev/null +++ b/test/plausible/google/search_console/filters_test.exs @@ -0,0 +1,183 @@ +defmodule Plausible.Google.SearchConsole.FiltersTest do + alias Plausible.Google.SearchConsole.Filters + use Plausible.DataCase, async: true + + test "transforms simple page filter" do + filters = %{ + "visit:entry_page" => {:is, "/page"} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{filters: [%{dimension: "page", expression: "https://plausible.io/page"}]} + ] + end + + test "transforms matches page filter" do + filters = %{ + "visit:entry_page" => {:matches, "*page*"} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{ + dimension: "page", + operator: "includingRegex", + expression: "^https://plausible\\.io.*page.*$" + } + ] + } + ] + end + + test "transforms member page filter" do + filters = %{ + "visit:entry_page" => {:member, ["/pageA", "/pageB"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{ + dimension: "page", + operator: "includingRegex", + expression: "https://plausible.io/pageA|https://plausible.io/pageB" + } + ] + } + ] + end + + test "transforms matches_member page filter" do + filters = %{ + "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{ + dimension: "page", + operator: "includingRegex", + expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$" + } + ] + } + ] + end + + test "transforms event:page exactly like visit:entry_page" do + filters = %{ + "event:page" => {:matches_member, ["/pageA*", "/pageB*"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{ + dimension: "page", + operator: "includingRegex", + expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$" + } + ] + } + ] + end + + test "transforms simple visit:screen filter" do + filters = %{ + "visit:screen" => {:is, "Desktop"} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [%{filters: [%{dimension: "device", expression: "DESKTOP"}]}] + end + + test "transforms member visit:screen filter" do + filters = %{ + "visit:screen" => {:member, ["Mobile", "Tablet"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{dimension: "device", operator: "includingRegex", expression: "Mobile|Tablet"} + ] + } + ] + end + + test "transforms simple visit:country filter to alpha3" do + filters = %{ + "visit:country" => {:is, "EE"} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [%{filters: [%{dimension: "country", expression: "EST"}]}] + end + + test "transforms member visit:country filter" do + filters = %{ + "visit:country" => {:member, ["EE", "PL"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{dimension: "country", operator: "includingRegex", expression: "EST|POL"} + ] + } + ] + end + + test "filters can be combined" do + filters = %{ + "visit:entry_page" => {:matches, "*web-analytics*"}, + "visit:screen" => {:is, "Desktop"}, + "visit:country" => {:member, ["EE", "PL"]} + } + + {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) + + assert transformed == [ + %{ + filters: [ + %{dimension: "device", expression: "DESKTOP"}, + %{ + dimension: "page", + operator: "includingRegex", + expression: "^https://plausible\\.io.*web\\-analytics.*$" + }, + %{dimension: "country", operator: "includingRegex", expression: "EST|POL"} + ] + } + ] + end + + test "when unsupported filter is included the whole set becomes invalid" do + filters = %{ + "visit:entry_page" => {:matches, "*web-analytics*"}, + "visit:screen" => {:is, "Desktop"}, + "visit:country" => {:member, ["EE", "PL"]}, + "visit:utm_medium" => {:is, "facebook"} + } + + assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters) + end +end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 4d4481c4c08e..dfa9480c02a0 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -1606,9 +1606,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end - test "gets keywords from Google", %{conn: conn, user: user, site: site} do - insert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com") - + test "gets keywords from Google", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, referrer_source: "DuckDuckGo", @@ -1627,10 +1625,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day") {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil) - assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "search_terms" => terms - } + assert json_response(conn, 200) == %{"search_terms" => terms} end test "works when filter expression is provided for source", %{ diff --git a/test/test_helper.exs b/test/test_helper.exs index 924b75a3b708..9b1b07f7a8f7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -23,7 +23,7 @@ if :minio in Keyword.fetch!(ExUnit.configuration(), :include) do end if Mix.env() == :ce_test do - IO.puts("Test mode: Communnity Edition") + IO.puts("Test mode: Community Edition") ExUnit.configure(exclude: [:slow, :minio, :ee_only]) else IO.puts("Test mode: Enterprise Edition")