From d4fde3a132fbf5037c97534c7cb41f501430d4d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:49:11 +0200 Subject: [PATCH 01/13] Ignore SQLite files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 21d1524..b0e37ea 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ error_tracker-*.tar /priv/static/app.css dev.local.exs +dev.db* From 17eb5448c1fcc0e67fa0992af9118a36016ee819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:49:45 +0200 Subject: [PATCH 02/13] Add SQLite migrator and migration --- lib/error_tracker/migration/sqlite.ex | 77 +++++++++++++++++++++++ lib/error_tracker/migration/sqlite/v01.ex | 43 +++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 lib/error_tracker/migration/sqlite.ex create mode 100644 lib/error_tracker/migration/sqlite/v01.ex diff --git a/lib/error_tracker/migration/sqlite.ex b/lib/error_tracker/migration/sqlite.ex new file mode 100644 index 0000000..5520613 --- /dev/null +++ b/lib/error_tracker/migration/sqlite.ex @@ -0,0 +1,77 @@ +defmodule ErrorTracker.Migration.SQLite do + @moduledoc false + + @behaviour ErrorTracker.Migration + + use Ecto.Migration + + import Ecto.Query + + @initial_version 1 + @current_version 1 + + @impl ErrorTracker.Migration + def up(opts) do + opts = with_defaults(opts, @current_version) + initial = current_version(opts) + + cond do + initial == 0 -> change(@initial_version..opts.version, :up) + initial < opts.version -> change((initial + 1)..opts.version, :up) + true -> :ok + end + end + + @impl ErrorTracker.Migration + def down(opts) do + opts = with_defaults(opts, @initial_version) + initial = max(current_version(opts), @initial_version) + + if initial >= opts.version do + change(initial..opts.version, :down) + end + end + + @impl ErrorTracker.Migration + def current_version(opts) do + opts = with_defaults(opts, @initial_version) + repo = Map.get_lazy(opts, :repo, fn -> repo() end) + + query = + from meta in "error_tracker_meta", + where: meta.key == "migration_version", + select: meta.value + + case repo.one(query, log: false) do + version when is_binary(version) -> String.to_integer(version) + _other -> 0 + end + rescue + # We get a Exqlite.Error error if the table doesn't exist yet - initial migration + Exqlite.Error -> 0 + end + + defp change(versions_range, direction) do + for version <- versions_range do + padded_version = String.pad_leading(to_string(version), 2, "0") + + migration_module = Module.concat(__MODULE__, "V#{padded_version}") + apply(migration_module, direction, []) + end + + case direction do + :up -> record_version(Enum.max(versions_range)) + :down -> record_version(Enum.min(versions_range) - 1) + end + end + + defp record_version(0), do: :ok + + defp record_version(version) do + execute "INSERT OR REPLACE INTO error_tracker_meta(key, value) VALUES('migration_version', '#{version}');" + end + + defp with_defaults(opts, version) do + Enum.into(opts, %{version: version}) + end +end diff --git a/lib/error_tracker/migration/sqlite/v01.ex b/lib/error_tracker/migration/sqlite/v01.ex new file mode 100644 index 0000000..44b02e7 --- /dev/null +++ b/lib/error_tracker/migration/sqlite/v01.ex @@ -0,0 +1,43 @@ +defmodule ErrorTracker.Migration.SQLite.V01 do + @moduledoc false + + use Ecto.Migration + + def up do + create table(:error_tracker_meta, primary_key: [name: :key, type: :string]) do + add :value, :string + end + + create table(:error_tracker_errors, primary_key: [name: :id, type: :bigserial]) do + add :kind, :string, null: false + add :reason, :text, null: false + add :source_line, :text, null: false + add :source_function, :text, null: false + add :status, :string, null: false + add :fingerprint, :string, null: false + add :last_occurrence_at, :utc_datetime_usec, null: false + + timestamps(type: :utc_datetime_usec) + end + + create unique_index(:error_tracker_errors, [:fingerprint]) + + create table(:error_tracker_occurrences, primary_key: [name: :id, type: :bigserial]) do + add :context, :map, null: false + add :reason, :text, null: false + add :stacktrace, :map, null: false + + add :error_id, references(:error_tracker_errors, on_delete: :delete_all, type: :bigserial), + null: false + + timestamps(type: :utc_datetime_usec, updated_at: false) + end + + create index(:error_tracker_occurrences, [:error_id]) + end + + def down do + drop table(:error_tracker_occurrences) + drop table(:error_tracker_errors) + end +end From c15dc82a4e6ce124b40eba44d67e109e1c7c0e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:49:57 +0200 Subject: [PATCH 03/13] Add SQLite3 optional dependency --- mix.exs | 3 ++- mix.lock | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d8fd93a..3bdbba4 100644 --- a/mix.exs +++ b/mix.exs @@ -83,7 +83,8 @@ defmodule ErrorTracker.MixProject do {:phoenix_live_view, "~> 0.19 or ~> 1.0"}, {:phoenix_ecto, "~> 4.6"}, {:plug, "~> 1.10"}, - {:postgrex, ">= 0.0.0"}, + {:postgrex, ">= 0.0.0", optional: true}, + {:ecto_sqlite3, ">= 0.0.0", optional: true}, # Dev dependencies {:bun, "~> 1.3", only: :dev}, {:credo, "~> 1.7", only: [:dev, :test]}, diff --git a/mix.lock b/mix.lock index e5beb83..9465977 100644 --- a/mix.lock +++ b/mix.lock @@ -2,6 +2,7 @@ "bun": {:hex, :bun, "1.3.0", "6833722da5b073777e043aec42091b0cf8bbacb84262ec6d348a914dda4c6a98", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "dde1b8116ba57269a9f398b4b28492b16fb29a78800c9533b7c9fb036793d62a"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, + "cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, @@ -11,7 +12,10 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "ecto_sql": {:hex, :ecto_sql, "3.11.2", "c7cc7f812af571e50b80294dc2e535821b3b795ce8008d07aa5f336591a185a8", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "73c07f995ac17dbf89d3cfaaf688fcefabcd18b7b004ac63b0dc4ef39499ed6b"}, + "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.16.0", "1cdc8ea6319e7cb1bc273a36db0ecde69ad56b4dea3037689ad8c0afc6a91e16", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.22", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "73c9dd56830d67c951bc254c082cb0a7f9fa139d44866bc3186c8859d1b4d787"}, + "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, "ex_doc": {:hex, :ex_doc, "0.33.0", "690562b153153c7e4d455dc21dab86e445f66ceba718defe64b0ef6f0bd83ba0", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "3f69adc28274cb51be37d09b03e4565232862a4b10288a3894587b0131412124"}, + "exqlite": {:hex, :exqlite, "0.23.0", "6e851c937a033299d0784994c66da24845415072adbc455a337e20087bce9033", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "404341cceec5e6466aaed160cf0b58be2019b60af82588c215e1224ebd3ec831"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, From 21c6ed5ac94f2ff016d56b6a318628a8837db104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:50:13 +0200 Subject: [PATCH 04/13] If the adapter is SQLite3, use the new migrator --- lib/error_tracker/migration.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/error_tracker/migration.ex b/lib/error_tracker/migration.ex index ffa768a..5d371b4 100644 --- a/lib/error_tracker/migration.ex +++ b/lib/error_tracker/migration.ex @@ -112,6 +112,7 @@ defmodule ErrorTracker.Migration do defp migrator do case ErrorTracker.Repo.__adapter__() do Ecto.Adapters.Postgres -> ErrorTracker.Migration.Postgres + Ecto.Adapters.SQLite3 -> ErrorTracker.Migration.SQLite adapter -> raise "ErrorTracker does not support #{adapter}" end end From 8186582c5a806ff804c487579bfa09d50f869dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:50:28 +0200 Subject: [PATCH 05/13] Do not use prefixes on non-PostgreSQL adapters --- lib/error_tracker/repo.ex | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/error_tracker/repo.ex b/lib/error_tracker/repo.ex index e243a6b..2b55b19 100644 --- a/lib/error_tracker/repo.ex +++ b/lib/error_tracker/repo.ex @@ -28,10 +28,20 @@ defmodule ErrorTracker.Repo do def __adapter__, do: repo().__adapter__() defp dispatch(action, args, opts) do - defaults = [prefix: Application.get_env(:error_tracker, :prefix, "public")] + repo = repo() + + defaults = + case repo.__adapter__() do + Ecto.Adapter.Postgresql -> + [prefix: Application.get_env(:error_tracker, :prefix, "public")] + + _ -> + [] + end + opts_w_defaults = Keyword.merge(defaults, opts) - apply(repo(), action, args ++ [opts_w_defaults]) + apply(repo, action, args ++ [opts_w_defaults]) end defp repo do From db6ccccc62c610958297c7d93bb2507c74255a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Mon, 5 Aug 2024 12:50:37 +0200 Subject: [PATCH 06/13] Update dev script and config --- dev.exs | 11 +++++++++-- dev.local.example.exs | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/dev.exs b/dev.exs index 08c33b1..6e194fa 100644 --- a/dev.exs +++ b/dev.exs @@ -13,11 +13,18 @@ Logger.configure(level: :debug) Code.require_file("dev.local.exs") # Prepare the repo + +adapter = + case Application.get_env(:error_tracker, :ecto_adapter) do + :postgres -> Ecto.Adapters.Postgres + :sqlite3 -> Ecto.Adapters.SQLite3 + end + defmodule ErrorTrackerDev.Repo do - use Ecto.Repo, otp_app: :error_tracker, adapter: Ecto.Adapters.Postgres + use Ecto.Repo, otp_app: :error_tracker, adapter: adapter end -_ = Ecto.Adapters.Postgres.storage_up(ErrorTrackerDev.Repo.config()) +_ = adapter.storage_up(ErrorTrackerDev.Repo.config()) # Configures the endpoint Application.put_env(:error_tracker, ErrorTrackerDevWeb.Endpoint, diff --git a/dev.local.example.exs b/dev.local.example.exs index 982fbaa..42464d1 100644 --- a/dev.local.example.exs +++ b/dev.local.example.exs @@ -1,6 +1,22 @@ -# Prepare the Repo URL +# PostgreSQL adapter +# +# To use SQLite3 on your local development machine uncomment these lines and +# comment the lines of other adapters. + +Application.put_env(:error_tracker, :ecto_adapter, :postgres) + Application.put_env( :error_tracker, ErrorTrackerDev.Repo, url: "ecto://postgres:postgres@127.0.0.1/error_tracker_dev" ) + +# SQlite3 adapter +# +# To use SQLite3 on your local development machine uncomment these lines and +# comment the lines of other adapters. + +# Application.put_env(:error_tracker, :ecto_adapter, :sqlite3) + +# sqlite_db = System.get_env("SQLITE_DB") || "dev.db" +# Application.put_env(:error_tracker, ErrorTrackerDev.Repo, database: sqlite_db) From 182dbc8d2ef9d89dc55847dd08b884148e297acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 11:39:50 +0200 Subject: [PATCH 07/13] Migrate Postgresql to use meta table --- lib/error_tracker/migration/postgres.ex | 42 +++++++---- lib/error_tracker/migration/postgres/v01.ex | 84 +++++++++++++-------- lib/error_tracker/migration/postgres/v02.ex | 21 ++++++ lib/error_tracker/migration/sqlite.ex | 13 ++-- 4 files changed, 109 insertions(+), 51 deletions(-) create mode 100644 lib/error_tracker/migration/postgres/v02.ex diff --git a/lib/error_tracker/migration/postgres.ex b/lib/error_tracker/migration/postgres.ex index 0ba9408..571623e 100644 --- a/lib/error_tracker/migration/postgres.ex +++ b/lib/error_tracker/migration/postgres.ex @@ -8,7 +8,7 @@ defmodule ErrorTracker.Migration.Postgres do import Ecto.Query @initial_version 1 - @current_version 1 + @current_version 2 @default_prefix "public" @impl ErrorTracker.Migration @@ -44,17 +44,14 @@ defmodule ErrorTracker.Migration.Postgres do repo = Map.get_lazy(opts, :repo, fn -> repo() end) query = - from pg_class in "pg_class", - left_join: pg_description in "pg_description", - on: pg_description.objoid == pg_class.oid, - left_join: pg_namespace in "pg_namespace", - on: pg_namespace.oid == pg_class.relnamespace, - where: pg_class.relname == "error_tracker_errors", - where: pg_namespace.nspname == ^opts.escaped_prefix, - select: pg_description.description - - case repo.one(query, log: false) do - version when is_binary(version) -> String.to_integer(version) + from meta in "error_tracker_meta", + where: meta.key == "migration_version", + select: meta.value + + with true <- meta_table_exists?(repo, opts), + version when is_binary(version) <- repo.one(query, log: false, prefix: opts[:prefix]) do + String.to_integer(version) + else _other -> 0 end end @@ -73,11 +70,13 @@ defmodule ErrorTracker.Migration.Postgres do end end + defp record_version(_opts, 0), do: :ok + defp record_version(%{prefix: prefix}, version) do - case version do - 0 -> :ok - _other -> execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS '#{version}'" - end + execute """ + INSERT INTO #{prefix}.error_tracker_meta (key, value) VALUES ('migration_version', '#{version}') + ON CONFLICT (key) DO UPDATE SET value = '#{version}' + """ end defp with_defaults(opts, version) do @@ -87,4 +86,15 @@ defmodule ErrorTracker.Migration.Postgres do |> Map.put_new(:create_schema, opts.prefix != @default_prefix) |> Map.put_new(:escaped_prefix, String.replace(opts.prefix, "'", "\\'")) end + + defp meta_table_exists?(repo, opts) do + Ecto.Adapters.SQL.query!( + repo, + "SELECT TRUE FROM information_schema.tables WHERE table_name = 'error_tracker_meta' AND table_schema = $1", + [opts.prefix], + log: false + ) + |> Map.get(:rows) + |> Enum.any?() + end end diff --git a/lib/error_tracker/migration/postgres/v01.ex b/lib/error_tracker/migration/postgres/v01.ex index 04e8189..903f4be 100644 --- a/lib/error_tracker/migration/postgres/v01.ex +++ b/lib/error_tracker/migration/postgres/v01.ex @@ -3,45 +3,69 @@ defmodule ErrorTracker.Migration.Postgres.V01 do use Ecto.Migration - def up(%{create_schema: create_schema, prefix: prefix}) do - if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}") - - create table(:error_tracker_errors, - primary_key: [name: :id, type: :bigserial], - prefix: prefix - ) do - add :kind, :string, null: false - add :reason, :text, null: false - add :source_line, :text, null: false - add :source_function, :text, null: false - add :status, :string, null: false - add :fingerprint, :string, null: false - add :last_occurrence_at, :utc_datetime_usec, null: false - - timestamps(type: :utc_datetime_usec) - end + import Ecto.Query - create unique_index(:error_tracker_errors, [:fingerprint], prefix: prefix) + def up(opts = %{create_schema: create_schema, prefix: prefix}) do + if current_version_legacy(opts) == 0 do + if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}") - create table(:error_tracker_occurrences, - primary_key: [name: :id, type: :bigserial], - prefix: prefix - ) do - add :context, :map, null: false - add :reason, :text, null: false - add :stacktrace, :map, null: false + create table(:error_tracker_errors, + primary_key: [name: :id, type: :bigserial], + prefix: prefix + ) do + add :kind, :string, null: false + add :reason, :text, null: false + add :source_line, :text, null: false + add :source_function, :text, null: false + add :status, :string, null: false + add :fingerprint, :string, null: false + add :last_occurrence_at, :utc_datetime_usec, null: false - add :error_id, references(:error_tracker_errors, on_delete: :delete_all, type: :bigserial), - null: false + timestamps(type: :utc_datetime_usec) + end - timestamps(type: :utc_datetime_usec, updated_at: false) - end + create unique_index(:error_tracker_errors, [:fingerprint], prefix: prefix) + + create table(:error_tracker_occurrences, + primary_key: [name: :id, type: :bigserial], + prefix: prefix + ) do + add :context, :map, null: false + add :reason, :text, null: false + add :stacktrace, :map, null: false + + add :error_id, + references(:error_tracker_errors, on_delete: :delete_all, type: :bigserial), + null: false + + timestamps(type: :utc_datetime_usec, updated_at: false) + end - create index(:error_tracker_occurrences, [:error_id], prefix: prefix) + create index(:error_tracker_occurrences, [:error_id], prefix: prefix) + else + :noop + end end def down(%{prefix: prefix}) do drop table(:error_tracker_occurrences, prefix: prefix) drop table(:error_tracker_errors, prefix: prefix) end + + def current_version_legacy(opts) do + query = + from pg_class in "pg_class", + left_join: pg_description in "pg_description", + on: pg_description.objoid == pg_class.oid, + left_join: pg_namespace in "pg_namespace", + on: pg_namespace.oid == pg_class.relnamespace, + where: pg_class.relname == "error_tracker_errors", + where: pg_namespace.nspname == ^opts.escaped_prefix, + select: pg_description.description + + case repo().one(query, log: false) do + version when is_binary(version) -> String.to_integer(version) + _other -> 0 + end + end end diff --git a/lib/error_tracker/migration/postgres/v02.ex b/lib/error_tracker/migration/postgres/v02.ex new file mode 100644 index 0000000..27c99c9 --- /dev/null +++ b/lib/error_tracker/migration/postgres/v02.ex @@ -0,0 +1,21 @@ +defmodule ErrorTracker.Migration.Postgres.V02 do + @moduledoc false + + use Ecto.Migration + + def up(%{prefix: prefix}) do + create table(:error_tracker_meta, + primary_key: [name: :key, type: :string], + prefix: prefix + ) do + add :value, :string, null: false + end + + execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS ''" + end + + def down(%{prefix: prefix}) do + drop table(:error_tracker_meta, prefix: prefix) + execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS '1'" + end +end diff --git a/lib/error_tracker/migration/sqlite.ex b/lib/error_tracker/migration/sqlite.ex index 5520613..84c4d61 100644 --- a/lib/error_tracker/migration/sqlite.ex +++ b/lib/error_tracker/migration/sqlite.ex @@ -42,13 +42,12 @@ defmodule ErrorTracker.Migration.SQLite do where: meta.key == "migration_version", select: meta.value - case repo.one(query, log: false) do - version when is_binary(version) -> String.to_integer(version) + with true <- meta_table_exists?(repo), + version when is_binary(version) <- repo.one(query, log: false) do + String.to_integer(version) + else _other -> 0 end - rescue - # We get a Exqlite.Error error if the table doesn't exist yet - initial migration - Exqlite.Error -> 0 end defp change(versions_range, direction) do @@ -74,4 +73,8 @@ defmodule ErrorTracker.Migration.SQLite do defp with_defaults(opts, version) do Enum.into(opts, %{version: version}) end + + defp meta_table_exists?(repo) do + Ecto.Adapters.SQL.table_exists?(repo, "error_tracker_meta", log: false) + end end From cf30a04e7fe161d290a2fd43807549bcb606047a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 11:45:39 +0200 Subject: [PATCH 08/13] Update documentation --- guides/Getting Started.md | 2 +- lib/error_tracker.ex | 3 ++- lib/error_tracker/migration.ex | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/guides/Getting Started.md b/guides/Getting Started.md index 3fe822d..64a7d5d 100644 --- a/guides/Getting Started.md +++ b/guides/Getting Started.md @@ -4,7 +4,7 @@ This guide is an introduction to ErrorTracker, an Elixir-based built-in error tr In this guide we will learn how to install ErrorTracker in an Elixir project so you can start reporting errors as soon as possible. We will also cover more advanced topics such as how to report custom errors and how to add extra context to reported errors. -**This guide requires you to have set up Ecto with PostgreSQL beforehand.** +**This guide requires you to have set up Ecto with PostgreSQL or SQLite3 beforehand.** ## Installing the ErrorTracker as a dependency diff --git a/lib/error_tracker.ex b/lib/error_tracker.ex index 5909368..f05a51b 100644 --- a/lib/error_tracker.ex +++ b/lib/error_tracker.ex @@ -16,7 +16,8 @@ defmodule ErrorTracker do ## Requirements - ErrorTracker requires Elixir 1.15+, Ecto 3.11+, Phoenix LiveView 0.19+, and PostgreSQL. + ErrorTracker requires Elixir 1.15+, Ecto 3.11+, Phoenix LiveView 0.19+, and + PostgreSQL or SQLite3 as database. ## Integrations diff --git a/lib/error_tracker/migration.ex b/lib/error_tracker/migration.ex index 5d371b4..dde5429 100644 --- a/lib/error_tracker/migration.ex +++ b/lib/error_tracker/migration.ex @@ -55,7 +55,7 @@ defmodule ErrorTracker.Migration do mix ecto.migrate ``` - ## Custom prefix + ## Custom prefix - PostgreSQL only ErrorTracker supports namespacing its own tables using PostgreSQL schemas, also known as "prefixes" in Ecto. With prefixes your error tables can reside outside of your primary From 25fc34eff136d8b54a54c6880c9c1b914337901a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 11:56:22 +0200 Subject: [PATCH 09/13] SQLite starts in version 2 --- lib/error_tracker/migration/sqlite.ex | 4 ++-- lib/error_tracker/migration/sqlite/{v01.ex => v02.ex} | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) rename lib/error_tracker/migration/sqlite/{v01.ex => v02.ex} (93%) diff --git a/lib/error_tracker/migration/sqlite.ex b/lib/error_tracker/migration/sqlite.ex index 84c4d61..5f7cbb0 100644 --- a/lib/error_tracker/migration/sqlite.ex +++ b/lib/error_tracker/migration/sqlite.ex @@ -7,8 +7,8 @@ defmodule ErrorTracker.Migration.SQLite do import Ecto.Query - @initial_version 1 - @current_version 1 + @initial_version 2 + @current_version 2 @impl ErrorTracker.Migration def up(opts) do diff --git a/lib/error_tracker/migration/sqlite/v01.ex b/lib/error_tracker/migration/sqlite/v02.ex similarity index 93% rename from lib/error_tracker/migration/sqlite/v01.ex rename to lib/error_tracker/migration/sqlite/v02.ex index 44b02e7..cc2745a 100644 --- a/lib/error_tracker/migration/sqlite/v01.ex +++ b/lib/error_tracker/migration/sqlite/v02.ex @@ -1,4 +1,4 @@ -defmodule ErrorTracker.Migration.SQLite.V01 do +defmodule ErrorTracker.Migration.SQLite.V02 do @moduledoc false use Ecto.Migration @@ -39,5 +39,6 @@ defmodule ErrorTracker.Migration.SQLite.V01 do def down do drop table(:error_tracker_occurrences) drop table(:error_tracker_errors) + drop table(:error_tracker_meta) end end From db7503c8e696fe5c7259186e41d2d41931a28d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 12:03:35 +0200 Subject: [PATCH 10/13] Add some comments --- lib/error_tracker/migration/postgres/v01.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/error_tracker/migration/postgres/v01.ex b/lib/error_tracker/migration/postgres/v01.ex index 903f4be..a67b4b9 100644 --- a/lib/error_tracker/migration/postgres/v01.ex +++ b/lib/error_tracker/migration/postgres/v01.ex @@ -6,6 +6,12 @@ defmodule ErrorTracker.Migration.Postgres.V01 do import Ecto.Query def up(opts = %{create_schema: create_schema, prefix: prefix}) do + # Prior to V02 the migration version was stored in table comments. + # As of now the migration version is stored in a new table (created in V02). + # + # However, systems migrating to V02 may think they need to run V01 too, so + # we need to check for the legacy version storage to avoid running this + # migration twice. if current_version_legacy(opts) == 0 do if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}") From 50e48dda3a73a5150958c7e9b603cfa4753143a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 18:10:52 +0200 Subject: [PATCH 11/13] Fix rollback --- lib/error_tracker/migration/postgres/v01.ex | 8 ++++++++ lib/error_tracker/migration/postgres/v02.ex | 14 +++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/error_tracker/migration/postgres/v01.ex b/lib/error_tracker/migration/postgres/v01.ex index a67b4b9..f55b8b3 100644 --- a/lib/error_tracker/migration/postgres/v01.ex +++ b/lib/error_tracker/migration/postgres/v01.ex @@ -15,6 +15,13 @@ defmodule ErrorTracker.Migration.Postgres.V01 do if current_version_legacy(opts) == 0 do if create_schema, do: execute("CREATE SCHEMA IF NOT EXISTS #{prefix}") + create table(:error_tracker_meta, + primary_key: [name: :key, type: :string], + prefix: prefix + ) do + add :value, :string, null: false + end + create table(:error_tracker_errors, primary_key: [name: :id, type: :bigserial], prefix: prefix @@ -56,6 +63,7 @@ defmodule ErrorTracker.Migration.Postgres.V01 do def down(%{prefix: prefix}) do drop table(:error_tracker_occurrences, prefix: prefix) drop table(:error_tracker_errors, prefix: prefix) + drop_if_exists table(:error_tracker_meta, prefix: prefix) end def current_version_legacy(opts) do diff --git a/lib/error_tracker/migration/postgres/v02.ex b/lib/error_tracker/migration/postgres/v02.ex index 27c99c9..81af264 100644 --- a/lib/error_tracker/migration/postgres/v02.ex +++ b/lib/error_tracker/migration/postgres/v02.ex @@ -4,10 +4,13 @@ defmodule ErrorTracker.Migration.Postgres.V02 do use Ecto.Migration def up(%{prefix: prefix}) do - create table(:error_tracker_meta, - primary_key: [name: :key, type: :string], - prefix: prefix - ) do + # For systems which executed versions without this migration they may not + # have the error_tracker_meta table, so we need to create it conditionally + # to avoid errors. + create_if_not_exists table(:error_tracker_meta, + primary_key: [name: :key, type: :string], + prefix: prefix + ) do add :value, :string, null: false end @@ -15,7 +18,8 @@ defmodule ErrorTracker.Migration.Postgres.V02 do end def down(%{prefix: prefix}) do - drop table(:error_tracker_meta, prefix: prefix) + # We do not delete the `error_tracker_meta` table because it's creation and + # deletion are controlled by V01 migration. execute "COMMENT ON TABLE #{inspect(prefix)}.error_tracker_errors IS '1'" end end From 6324adc9e0841ac30c24d8ad8fa5bbe5e96ca9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 18:11:34 +0200 Subject: [PATCH 12/13] Not null values on meta table --- lib/error_tracker/migration/sqlite/v02.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/error_tracker/migration/sqlite/v02.ex b/lib/error_tracker/migration/sqlite/v02.ex index cc2745a..feda9c0 100644 --- a/lib/error_tracker/migration/sqlite/v02.ex +++ b/lib/error_tracker/migration/sqlite/v02.ex @@ -5,7 +5,7 @@ defmodule ErrorTracker.Migration.SQLite.V02 do def up do create table(:error_tracker_meta, primary_key: [name: :key, type: :string]) do - add :value, :string + add :value, :string, null: false end create table(:error_tracker_errors, primary_key: [name: :id, type: :bigserial]) do From c5f6f14118aadb1078f7f1af5aacbd7f5aa17863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93scar=20de=20Arriba?= Date: Tue, 6 Aug 2024 18:44:02 +0200 Subject: [PATCH 13/13] Unify migrators --- lib/error_tracker/migration/postgres.ex | 70 +-------------- lib/error_tracker/migration/sql_migrator.ex | 99 +++++++++++++++++++++ lib/error_tracker/migration/sqlite.ex | 55 +----------- lib/error_tracker/migration/sqlite/v02.ex | 4 +- 4 files changed, 109 insertions(+), 119 deletions(-) create mode 100644 lib/error_tracker/migration/sql_migrator.ex diff --git a/lib/error_tracker/migration/postgres.ex b/lib/error_tracker/migration/postgres.ex index 571623e..e232587 100644 --- a/lib/error_tracker/migration/postgres.ex +++ b/lib/error_tracker/migration/postgres.ex @@ -4,8 +4,7 @@ defmodule ErrorTracker.Migration.Postgres do @behaviour ErrorTracker.Migration use Ecto.Migration - - import Ecto.Query + alias ErrorTracker.Migration.SQLMigrator @initial_version 1 @current_version 2 @@ -14,69 +13,19 @@ defmodule ErrorTracker.Migration.Postgres do @impl ErrorTracker.Migration def up(opts) do opts = with_defaults(opts, @current_version) - initial = current_version(opts) - - cond do - initial == 0 -> - change(@initial_version..opts.version, :up, opts) - - initial < opts.version -> - change((initial + 1)..opts.version, :up, opts) - - true -> - :ok - end + SQLMigrator.migrate_up(__MODULE__, opts, @initial_version) end @impl ErrorTracker.Migration def down(opts) do opts = with_defaults(opts, @initial_version) - initial = max(current_version(opts), @initial_version) - - if initial >= opts.version do - change(initial..opts.version, :down, opts) - end + SQLMigrator.migrate_down(__MODULE__, opts, @initial_version) end @impl ErrorTracker.Migration def current_version(opts) do opts = with_defaults(opts, @initial_version) - repo = Map.get_lazy(opts, :repo, fn -> repo() end) - - query = - from meta in "error_tracker_meta", - where: meta.key == "migration_version", - select: meta.value - - with true <- meta_table_exists?(repo, opts), - version when is_binary(version) <- repo.one(query, log: false, prefix: opts[:prefix]) do - String.to_integer(version) - else - _other -> 0 - end - end - - defp change(versions_range, direction, opts) do - for version <- versions_range do - padded_version = String.pad_leading(to_string(version), 2, "0") - - migration_module = Module.concat(__MODULE__, "V#{padded_version}") - apply(migration_module, direction, [opts]) - end - - case direction do - :up -> record_version(opts, Enum.max(versions_range)) - :down -> record_version(opts, Enum.min(versions_range) - 1) - end - end - - defp record_version(_opts, 0), do: :ok - - defp record_version(%{prefix: prefix}, version) do - execute """ - INSERT INTO #{prefix}.error_tracker_meta (key, value) VALUES ('migration_version', '#{version}') - ON CONFLICT (key) DO UPDATE SET value = '#{version}' - """ + SQLMigrator.current_version(opts) end defp with_defaults(opts, version) do @@ -86,15 +35,4 @@ defmodule ErrorTracker.Migration.Postgres do |> Map.put_new(:create_schema, opts.prefix != @default_prefix) |> Map.put_new(:escaped_prefix, String.replace(opts.prefix, "'", "\\'")) end - - defp meta_table_exists?(repo, opts) do - Ecto.Adapters.SQL.query!( - repo, - "SELECT TRUE FROM information_schema.tables WHERE table_name = 'error_tracker_meta' AND table_schema = $1", - [opts.prefix], - log: false - ) - |> Map.get(:rows) - |> Enum.any?() - end end diff --git a/lib/error_tracker/migration/sql_migrator.ex b/lib/error_tracker/migration/sql_migrator.ex new file mode 100644 index 0000000..33d231f --- /dev/null +++ b/lib/error_tracker/migration/sql_migrator.ex @@ -0,0 +1,99 @@ +defmodule ErrorTracker.Migration.SQLMigrator do + @moduledoc false + + use Ecto.Migration + + import Ecto.Query + + def migrate_up(migrator, opts, initial_version) do + initial = current_version(opts) + + cond do + initial == 0 -> + change(migrator, initial_version..opts.version, :up, opts) + + initial < opts.version -> + change(migrator, (initial + 1)..opts.version, :up, opts) + + true -> + :ok + end + end + + def migrate_down(migrator, opts, initial_version) do + initial = max(current_version(opts), initial_version) + + if initial >= opts.version do + change(migrator, initial..opts.version, :down, opts) + end + end + + def current_version(opts) do + repo = Map.get_lazy(opts, :repo, fn -> repo() end) + + query = + from meta in "error_tracker_meta", + where: meta.key == "migration_version", + select: meta.value + + with true <- meta_table_exists?(repo, opts), + version when is_binary(version) <- repo.one(query, log: false, prefix: opts[:prefix]) do + String.to_integer(version) + else + _other -> 0 + end + end + + defp change(migrator, versions_range, direction, opts) do + for version <- versions_range do + padded_version = String.pad_leading(to_string(version), 2, "0") + + migration_module = Module.concat(migrator, "V#{padded_version}") + apply(migration_module, direction, [opts]) + end + + case direction do + :up -> record_version(opts, Enum.max(versions_range)) + :down -> record_version(opts, Enum.min(versions_range) - 1) + end + end + + defp record_version(_opts, 0), do: :ok + + defp record_version(%{prefix: prefix}, version) do + timestamp = DateTime.utc_now() |> DateTime.to_unix() + + case repo().__adapter__() do + Ecto.Adapters.Postgres -> + execute """ + INSERT INTO #{prefix}.error_tracker_meta (key, value) + VALUES ('migration_version', '#{version}'), ('migration_timestamp', #{timestamp}) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value + """ + + _other -> + execute """ + INSERT INTO error_tracker_meta (key, value) + VALUES ('migration_version', '#{version}'), ('migration_timestamp', #{timestamp}) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value + """ + end + end + + defp meta_table_exists?(repo, opts) do + case repo.__adapter__() do + Ecto.Adapters.Postgres -> + Ecto.Adapters.SQL.query!( + repo, + "SELECT TRUE FROM information_schema.tables WHERE table_name = 'error_tracker_meta' AND table_schema = $1", + [opts.prefix], + log: false + ) + |> Map.get(:rows) + |> Enum.any?() + + _other -> + Ecto.Adapters.SQL.table_exists?(repo, "error_tracker_meta", log: false) + end + end +end diff --git a/lib/error_tracker/migration/sqlite.ex b/lib/error_tracker/migration/sqlite.ex index 5f7cbb0..84861c5 100644 --- a/lib/error_tracker/migration/sqlite.ex +++ b/lib/error_tracker/migration/sqlite.ex @@ -4,8 +4,7 @@ defmodule ErrorTracker.Migration.SQLite do @behaviour ErrorTracker.Migration use Ecto.Migration - - import Ecto.Query + alias ErrorTracker.Migration.SQLMigrator @initial_version 2 @current_version 2 @@ -13,68 +12,22 @@ defmodule ErrorTracker.Migration.SQLite do @impl ErrorTracker.Migration def up(opts) do opts = with_defaults(opts, @current_version) - initial = current_version(opts) - - cond do - initial == 0 -> change(@initial_version..opts.version, :up) - initial < opts.version -> change((initial + 1)..opts.version, :up) - true -> :ok - end + SQLMigrator.migrate_up(__MODULE__, opts, @initial_version) end @impl ErrorTracker.Migration def down(opts) do opts = with_defaults(opts, @initial_version) - initial = max(current_version(opts), @initial_version) - - if initial >= opts.version do - change(initial..opts.version, :down) - end + SQLMigrator.migrate_down(__MODULE__, opts, @initial_version) end @impl ErrorTracker.Migration def current_version(opts) do opts = with_defaults(opts, @initial_version) - repo = Map.get_lazy(opts, :repo, fn -> repo() end) - - query = - from meta in "error_tracker_meta", - where: meta.key == "migration_version", - select: meta.value - - with true <- meta_table_exists?(repo), - version when is_binary(version) <- repo.one(query, log: false) do - String.to_integer(version) - else - _other -> 0 - end - end - - defp change(versions_range, direction) do - for version <- versions_range do - padded_version = String.pad_leading(to_string(version), 2, "0") - - migration_module = Module.concat(__MODULE__, "V#{padded_version}") - apply(migration_module, direction, []) - end - - case direction do - :up -> record_version(Enum.max(versions_range)) - :down -> record_version(Enum.min(versions_range) - 1) - end - end - - defp record_version(0), do: :ok - - defp record_version(version) do - execute "INSERT OR REPLACE INTO error_tracker_meta(key, value) VALUES('migration_version', '#{version}');" + SQLMigrator.current_version(opts) end defp with_defaults(opts, version) do Enum.into(opts, %{version: version}) end - - defp meta_table_exists?(repo) do - Ecto.Adapters.SQL.table_exists?(repo, "error_tracker_meta", log: false) - end end diff --git a/lib/error_tracker/migration/sqlite/v02.ex b/lib/error_tracker/migration/sqlite/v02.ex index feda9c0..9d76fc3 100644 --- a/lib/error_tracker/migration/sqlite/v02.ex +++ b/lib/error_tracker/migration/sqlite/v02.ex @@ -3,7 +3,7 @@ defmodule ErrorTracker.Migration.SQLite.V02 do use Ecto.Migration - def up do + def up(_opts) do create table(:error_tracker_meta, primary_key: [name: :key, type: :string]) do add :value, :string, null: false end @@ -36,7 +36,7 @@ defmodule ErrorTracker.Migration.SQLite.V02 do create index(:error_tracker_occurrences, [:error_id]) end - def down do + def down(_opts) do drop table(:error_tracker_occurrences) drop table(:error_tracker_errors) drop table(:error_tracker_meta)