From bafb32b326bde7fa96e1afc23c52f269bba7bf20 Mon Sep 17 00:00:00 2001 From: zafer Date: Wed, 19 Mar 2025 16:56:31 +0200 Subject: [PATCH 1/5] feat: global commands --- .formatter.exs | 2 +- lib/algora/application.ex | 2 +- lib/algora/integrations/github/client.ex | 58 +++++- .../integrations/github/poller/search.ex | 180 ++++++++++++++++++ .../github/poller/search_consumer.ex | 66 +++++++ lib/algora/search/schemas/search_cursor.ex | 23 +++ lib/algora/search/search.ex | 34 ++++ lib/algora_web/live/home_live.ex | 15 +- mix.exs | 5 +- .../20250319144614_create_search_cursors.exs | 15 ++ 10 files changed, 387 insertions(+), 13 deletions(-) create mode 100644 lib/algora/integrations/github/poller/search.ex create mode 100644 lib/algora/integrations/github/poller/search_consumer.ex create mode 100644 lib/algora/search/schemas/search_cursor.ex create mode 100644 lib/algora/search/search.ex create mode 100644 priv/repo/migrations/20250319144614_create_search_cursors.exs diff --git a/.formatter.exs b/.formatter.exs index d487aae7d..7369d4e36 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,6 +1,6 @@ [ import_deps: [:ecto, :ecto_sql, :phoenix], subdirectories: ["priv/*/migrations", "config"], - plugins: [Phoenix.LiveView.HTMLFormatter, Styler], + # plugins: [Phoenix.LiveView.HTMLFormatter, Styler], inputs: ["*.{heex,ex,exs}", "{scripts,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] ] diff --git a/lib/algora/application.ex b/lib/algora/application.ex index 74c25bfc0..67242ebb5 100644 --- a/lib/algora/application.ex +++ b/lib/algora/application.ex @@ -21,9 +21,9 @@ defmodule Algora.Application do {Finch, name: Algora.Finch}, Algora.Github.TokenPool, Algora.Github.Poller.RootSupervisor, - Algora.Stargazer, # Start to serve requests, typically the last entry AlgoraWeb.Endpoint, + Algora.Stargazer, TwMerge.Cache ] diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index a14a3b6c6..69464deaf 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -34,7 +34,9 @@ defmodule Algora.Github.Client do {:ok, %Finch.Response{status: status, headers: headers}} when status in [301, 302, 307] -> case List.keyfind(headers, "location", 0) do {"location", location} -> - request_with_follow_redirects(Finch.build(request.method, location, request.headers, request.body)) + request_with_follow_redirects( + Finch.build(request.method, location, request.headers, request.body) + ) nil -> {:error, "Redirect response missing location header"} @@ -99,7 +101,8 @@ defmodule Algora.Github.Client do def fetch(access_token, url, method \\ "GET", body \\ nil) - def fetch(access_token, "https://api.github.com" <> path, method, body), do: fetch(access_token, path, method, body) + def fetch(access_token, "https://api.github.com" <> path, method, body), + do: fetch(access_token, path, method, body) def fetch(access_token, path, method, body) do http( @@ -112,6 +115,50 @@ defmodule Algora.Github.Client do ) end + def search(access_token, q, opts \\ []) do + per_page = opts[:per_page] || 10 + since = opts[:since] || "2025-03-18T00:00:00Z" + + search_query = + if opts[:since] do + "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc updated:>#{opts[:since]}" + else + "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc" + end + + query = """ + query issues($search_query: String!) { + search(first: #{per_page}, type: ISSUE, query: $search_query) { + issueCount + pageInfo { + hasNextPage + } + nodes { + __typename + ... on Issue { + url + updatedAt + comments(first: 3, orderBy: {field: UPDATED_AT, direction: ASC}) { + nodes { + databaseId + author { + login + } + body + } + } + } + } + } + } + """ + + body = %{query: query, variables: %{search_query: search_query}} + IO.puts(query) + IO.puts(search_query) + fetch(access_token, "/graphql", "POST", body) + end + defp build_query(opts), do: if(opts == [], do: "", else: "?" <> URI.encode_query(opts)) @impl true @@ -205,7 +252,8 @@ defmodule Algora.Github.Client do @impl true def list_installation_repos(access_token) do - with {:ok, %{"repositories" => repos}} <- fetch(access_token, "/installation/repositories", "GET") do + with {:ok, %{"repositories" => repos}} <- + fetch(access_token, "/installation/repositories", "GET") do {:ok, repos} end end @@ -242,6 +290,8 @@ defmodule Algora.Github.Client do @impl true def add_labels(access_token, owner, repo, number, labels) do - fetch(access_token, "/repos/#{owner}/#{repo}/issues/#{number}/labels", "POST", %{labels: labels}) + fetch(access_token, "/repos/#{owner}/#{repo}/issues/#{number}/labels", "POST", %{ + labels: labels + }) end end diff --git a/lib/algora/integrations/github/poller/search.ex b/lib/algora/integrations/github/poller/search.ex new file mode 100644 index 000000000..ef4b34678 --- /dev/null +++ b/lib/algora/integrations/github/poller/search.ex @@ -0,0 +1,180 @@ +defmodule Algora.Github.Poller.Search do + @moduledoc false + use GenServer + + import Ecto.Query, warn: false + + alias Algora.Search + alias Algora.Github + alias Algora.Github.Command + alias Algora.Parser + alias Algora.Admin + alias Algora.Repo + alias Algora.Util + + require Logger + + @per_page 10 + @poll_interval :timer.seconds(3) + + # Client API + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + def pause(pid) do + GenServer.cast(pid, :pause) + end + + def resume(pid) do + GenServer.cast(pid, :resume) + end + + # Server callbacks + @impl true + def init(opts) do + {:ok, + %{ + cursor: nil, + paused: not Algora.config([:auto_start_pollers]) + }, {:continue, :setup}} + end + + @impl true + def handle_continue(:setup, state) do + {:ok, cursor} = get_or_create_cursor() + schedule_poll() + + {:noreply, %{state | cursor: cursor}} + end + + @impl true + def handle_info(:poll, %{paused: true} = state) do + {:noreply, state} + end + + @impl true + def handle_info(:poll, state) do + {:ok, new_state} = poll(state) + schedule_poll() + {:noreply, new_state} + end + + @impl true + def handle_cast(:pause, state) do + {:noreply, %{state | paused: true}} + end + + @impl true + def handle_cast(:resume, state) do + schedule_poll() + {:noreply, %{state | paused: false}} + end + + @impl true + def handle_call(:get_repo_info, _from, state) do + {:reply, {state.repo_owner, state.repo_name}, state} + end + + @impl true + def handle_call(:is_paused, _from, state) do + {:reply, state.paused, state} + end + + defp schedule_poll do + Process.send_after(self(), :poll, @poll_interval) + end + + def poll(state) do + with {:ok, tickets} <- fetch_tickets(state), + if(length(tickets) > 0, do: Logger.debug("Processing #{length(tickets)} tickets")), + {:ok, updated_cursor} <- process_batch(tickets, state.cursor) do + {:ok, %{state | cursor: updated_cursor}} + else + {:error, reason} -> + Logger.error("Failed to fetch tickets: #{inspect(reason)}") + {:ok, state} + end + end + + defp process_batch([], search_cursor), do: {:ok, search_cursor} + + defp process_batch(tickets, search_cursor) do + Repo.transact(fn -> + with :ok <- process_tickets(tickets) do + update_last_polled(search_cursor, List.last(tickets)) + end + end) + end + + defp process_tickets(tickets) do + Enum.reduce_while(tickets, :ok, fn ticket, _acc -> + case process_ticket(ticket) do + {:ok, _} -> {:cont, :ok} + error -> {:halt, error} + end + end) + end + + def fetch_tickets(state) do + Github.Client.search(Admin.token!(), "bounty", + since: DateTime.to_iso8601(state.cursor.timestamp) + ) + end + + defp get_or_create_cursor() do + case Search.get_search_cursor("github") do + nil -> + Search.create_search_cursor(%{provider: "github", timestamp: DateTime.utc_now()}) + + search_cursor -> + {:ok, search_cursor} + end + end + + defp update_last_polled(search_cursor, %{"id" => id, "updated_at" => updated_at}) do + with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), + {:ok, cursor} <- + Search.update_search_cursor(search_cursor, %{ + timestamp: updated_at, + last_polled_at: DateTime.utc_now() + }) do + {:ok, cursor} + else + {:error, reason} -> Logger.error("Failed to update search cursor: #{inspect(reason)}") + end + end + + defp process_ticket( + %{"updated_at" => updated_at, "body" => body, "html_url" => html_url} = ticket + ) do + with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), + {:ok, [ticket_ref: ticket_ref], _, _, _, _} <- Parser.full_ticket_ref(html_url), + {:ok, commands} <- Command.parse(body) do + Logger.info("Latency: #{DateTime.diff(DateTime.utc_now(), updated_at, :second)}s") + + Enum.reduce_while(commands, :ok, fn command, _acc -> + res = + %{ + ticket: ticket, + command: Util.term_to_base64(command), + ticket_ref: Util.term_to_base64(ticket_ref) + } + |> Github.Poller.CommentConsumer.new() + |> Oban.insert() + + case res do + {:ok, _job} -> {:cont, :ok} + error -> {:halt, error} + end + end) + else + {:error, reason} -> + Logger.error( + "Failed to parse commands from ticket: #{inspect(ticket)}. Reason: #{inspect(reason)}" + ) + + :ok + end + end +end diff --git a/lib/algora/integrations/github/poller/search_consumer.ex b/lib/algora/integrations/github/poller/search_consumer.ex new file mode 100644 index 000000000..a91756ae6 --- /dev/null +++ b/lib/algora/integrations/github/poller/search_consumer.ex @@ -0,0 +1,66 @@ +defmodule Algora.Github.Poller.SearchConsumer do + @moduledoc false + use Oban.Worker, queue: :search_consumers + + alias Algora.Accounts + alias Algora.Bounties + alias Algora.Util + + require Logger + + @impl Oban.Worker + def perform(%Oban.Job{ + args: + %{ + "comment" => comment, + "command" => encoded_command, + "ticket_ref" => encoded_ticket_ref + } = _args + }) do + command = Util.base64_to_term!(encoded_command) + ticket_ref = Util.base64_to_term!(encoded_ticket_ref) + + run_command(command, ticket_ref, comment) + end + + defp run_command({:claim, _args}, _ticket_ref, _comment) do + # TODO: implement claim command + :ok + end + + defp run_command({:tip, args}, ticket_ref, _comment) do + Bounties.create_tip_intent(%{ + recipient: args[:recipient], + amount: args[:amount], + ticket_ref: %{ + owner: ticket_ref[:owner], + repo: ticket_ref[:repo], + number: ticket_ref[:number] + } + }) + end + + defp run_command({:bounty, args}, ticket_ref, comment) do + case Accounts.fetch_user_by(provider_id: to_string(comment["user"]["id"])) do + {:ok, user} -> + Bounties.create_bounty( + %{ + creator: user, + owner: user, + amount: args[:amount], + ticket_ref: %{ + owner: ticket_ref[:owner], + repo: ticket_ref[:repo], + number: ticket_ref[:number] + } + }, + command_id: comment["id"], + command_source: :comment + ) + + {:error, _reason} = error -> + Logger.error("Failed to create bounty: #{inspect(error)}") + error + end + end +end diff --git a/lib/algora/search/schemas/search_cursor.ex b/lib/algora/search/schemas/search_cursor.ex new file mode 100644 index 000000000..034551a36 --- /dev/null +++ b/lib/algora/search/schemas/search_cursor.ex @@ -0,0 +1,23 @@ +defmodule Algora.Search.SearchCursor do + @moduledoc false + use Algora.Schema + + import Ecto.Changeset + + typed_schema "search_cursors" do + field :provider, :string + field :timestamp, :utc_datetime_usec + field :last_polled_at, :utc_datetime_usec + + timestamps() + end + + @doc false + def changeset(search_cursor, attrs) do + search_cursor + |> cast(attrs, [:provider, :timestamp, :last_polled_at]) + |> generate_id() + |> validate_required([:provider, :timestamp]) + |> unique_constraint([:provider]) + end +end diff --git a/lib/algora/search/search.ex b/lib/algora/search/search.ex new file mode 100644 index 000000000..4063db5fa --- /dev/null +++ b/lib/algora/search/search.ex @@ -0,0 +1,34 @@ +defmodule Algora.Search do + @moduledoc false + import Ecto.Query, warn: false + + alias Algora.Search.SearchCursor + alias Algora.Repo + + def get_search_cursor(provider) do + Repo.get_by(SearchCursor, provider: provider) + end + + def delete_search_cursor(provider) do + case get_search_cursor(provider) do + nil -> {:error, :cursor_not_found} + cursor -> Repo.delete(cursor) + end + end + + def create_search_cursor(attrs \\ %{}) do + %SearchCursor{} + |> SearchCursor.changeset(attrs) + |> Repo.insert() + end + + def update_search_cursor(%SearchCursor{} = search_cursor, attrs) do + search_cursor + |> SearchCursor.changeset(attrs) + |> Repo.update() + end + + def list_cursors do + Repo.all(from(p in SearchCursor)) + end +end diff --git a/lib/algora_web/live/home_live.ex b/lib/algora_web/live/home_live.ex index cabbc24ef..cc28d3536 100644 --- a/lib/algora_web/live/home_live.ex +++ b/lib/algora_web/live/home_live.ex @@ -965,11 +965,12 @@ defmodule AlgoraWeb.HomeLive do defp get_total_paid_out do subtotal = Repo.one( - from t in Transaction, + from(t in Transaction, where: t.type == :credit, where: t.status == :succeeded, where: not is_nil(t.linked_transaction_id), select: sum(t.net_amount) + ) ) || Money.new(0, :USD) subtotal |> Money.add!(PlatformStats.get().extra_paid_out) |> Money.round(currency_digits: 0) @@ -978,22 +979,24 @@ defmodule AlgoraWeb.HomeLive do defp get_completed_bounties_count do bounties_subtotal = Repo.one( - from t in Transaction, + from(t in Transaction, where: t.type == :credit, where: t.status == :succeeded, where: not is_nil(t.linked_transaction_id), where: not is_nil(t.bounty_id), select: count(fragment("DISTINCT (?, ?)", t.bounty_id, t.user_id)) + ) ) || 0 tips_subtotal = Repo.one( - from t in Transaction, + from(t in Transaction, where: t.type == :credit, where: t.status == :succeeded, where: not is_nil(t.linked_transaction_id), where: not is_nil(t.tip_id), select: count(fragment("DISTINCT (?, ?)", t.tip_id, t.user_id)) + ) ) || 0 bounties_subtotal + tips_subtotal + PlatformStats.get().extra_completed_bounties @@ -1002,11 +1005,12 @@ defmodule AlgoraWeb.HomeLive do defp get_contributors_count do subtotal = Repo.one( - from t in Transaction, + from(t in Transaction, where: t.type == :credit, where: t.status == :succeeded, where: not is_nil(t.linked_transaction_id), select: count(fragment("DISTINCT ?", t.user_id)) + ) ) || 0 subtotal + PlatformStats.get().extra_contributors @@ -1014,7 +1018,7 @@ defmodule AlgoraWeb.HomeLive do defp get_countries_count do Repo.one( - from u in User, + from(u in User, join: t in Transaction, on: t.user_id == u.id, where: t.type == :credit, @@ -1022,6 +1026,7 @@ defmodule AlgoraWeb.HomeLive do where: not is_nil(t.linked_transaction_id), where: not is_nil(u.country) and u.country != "", select: count(fragment("DISTINCT ?", u.country)) + ) ) || 0 end diff --git a/mix.exs b/mix.exs index fd5d2ca41..b3454783b 100644 --- a/mix.exs +++ b/mix.exs @@ -56,7 +56,8 @@ defmodule Algora.MixProject do {:floki, ">= 0.30.0"}, {:phoenix_live_dashboard, "~> 0.8.3"}, {:tailwind, "~> 0.2", runtime: Mix.env() == :dev}, - {:tabler_icons, github: "algora-io/icons", sparse: "icons", app: false, compile: false, depth: 1}, + {:tabler_icons, + github: "algora-io/icons", sparse: "icons", app: false, compile: false, depth: 1}, {:swoosh, "~> 1.5"}, {:finch, "~> 0.13"}, {:httpoison, "~> 2.2"}, @@ -82,6 +83,7 @@ defmodule Algora.MixProject do {:live_svelte, "~> 0.14.1"}, {:nimble_parsec, "~> 1.4"}, {:oban, "~> 2.19"}, + {:oban_web, "~> 2.11"}, {:styler, "~> 1.2", only: [:dev, :test], runtime: false}, {:typed_ecto_schema, "~> 0.4.1", runtime: false}, {:chameleon, "~> 2.2.0"}, @@ -97,7 +99,6 @@ defmodule Algora.MixProject do # monitoring, logging {:appsignal_phoenix, "~> 2.6"}, {:logfmt_ex, "~> 0.4"}, - {:oban_web, "~> 2.11"}, # TODO: delete after migration {:yaml_elixir, "~> 2.9"} ] diff --git a/priv/repo/migrations/20250319144614_create_search_cursors.exs b/priv/repo/migrations/20250319144614_create_search_cursors.exs new file mode 100644 index 000000000..2e2bede8c --- /dev/null +++ b/priv/repo/migrations/20250319144614_create_search_cursors.exs @@ -0,0 +1,15 @@ +defmodule Algora.Repo.Migrations.CreateSearchCursors do + use Ecto.Migration + + def change do + create table(:search_cursors) do + add :provider, :string, null: false + add :timestamp, :utc_datetime_usec, null: false + add :last_polled_at, :utc_datetime_usec + + timestamps() + end + + create unique_index(:search_cursors, [:provider]) + end +end From e40ca9d5909b1604cf1062fefb9dbd8f62426667 Mon Sep 17 00:00:00 2001 From: zafer Date: Wed, 19 Mar 2025 17:44:20 +0200 Subject: [PATCH 2/5] feat: add search consumer and update GitHub poller - Introduced a new `search_consumers` queue in the configuration. - Removed the deprecated `search` function from the GitHub client. - Updated the `SearchConsumer` to handle command processing with GitHub comments. - Refactored the `Search` module to include a new search function for fetching tickets. - Adjusted the `Supervisor` to manage search consumers and their respective providers. --- config/config.exs | 1 + lib/algora/integrations/github/client.ex | 44 ---------- .../integrations/github/poller/search.ex | 82 +++++++++++++++---- .../github/poller/search_consumer.ex | 7 +- .../integrations/github/poller/supervisor.ex | 57 ++++++------- 5 files changed, 99 insertions(+), 92 deletions(-) diff --git a/config/config.exs b/config/config.exs index 81681785b..39781e7c9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -40,6 +40,7 @@ config :algora, Oban, queues: [ event_consumers: 1, comment_consumers: 1, + search_consumers: 1, github_og_image: 5, notify_bounty: 1, notify_tip_intent: 1, diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index 69464deaf..477eea116 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -115,50 +115,6 @@ defmodule Algora.Github.Client do ) end - def search(access_token, q, opts \\ []) do - per_page = opts[:per_page] || 10 - since = opts[:since] || "2025-03-18T00:00:00Z" - - search_query = - if opts[:since] do - "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc updated:>#{opts[:since]}" - else - "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc" - end - - query = """ - query issues($search_query: String!) { - search(first: #{per_page}, type: ISSUE, query: $search_query) { - issueCount - pageInfo { - hasNextPage - } - nodes { - __typename - ... on Issue { - url - updatedAt - comments(first: 3, orderBy: {field: UPDATED_AT, direction: ASC}) { - nodes { - databaseId - author { - login - } - body - } - } - } - } - } - } - """ - - body = %{query: query, variables: %{search_query: search_query}} - IO.puts(query) - IO.puts(search_query) - fetch(access_token, "/graphql", "POST", body) - end - defp build_query(opts), do: if(opts == [], do: "", else: "?" <> URI.encode_query(opts)) @impl true diff --git a/lib/algora/integrations/github/poller/search.ex b/lib/algora/integrations/github/poller/search.ex index ef4b34678..c030acc39 100644 --- a/lib/algora/integrations/github/poller/search.ex +++ b/lib/algora/integrations/github/poller/search.ex @@ -33,8 +33,11 @@ defmodule Algora.Github.Poller.Search do # Server callbacks @impl true def init(opts) do + provider = Keyword.fetch!(opts, :provider) + {:ok, %{ + provider: provider, cursor: nil, paused: not Algora.config([:auto_start_pollers]) }, {:continue, :setup}} @@ -72,8 +75,8 @@ defmodule Algora.Github.Poller.Search do end @impl true - def handle_call(:get_repo_info, _from, state) do - {:reply, {state.repo_owner, state.repo_name}, state} + def handle_call(:get_provider, _from, state) do + {:reply, state.provider, state} end @impl true @@ -117,9 +120,13 @@ defmodule Algora.Github.Poller.Search do end def fetch_tickets(state) do - Github.Client.search(Admin.token!(), "bounty", - since: DateTime.to_iso8601(state.cursor.timestamp) - ) + case search("bounty", since: DateTime.to_iso8601(state.cursor.timestamp)) do + {:ok, %{"data" => %{"search" => %{"nodes" => tickets}}}} -> + {:ok, tickets} + + _ -> + {:error, :no_tickets_found} + end end defp get_or_create_cursor() do @@ -132,7 +139,7 @@ defmodule Algora.Github.Poller.Search do end end - defp update_last_polled(search_cursor, %{"id" => id, "updated_at" => updated_at}) do + defp update_last_polled(search_cursor, %{"updatedAt" => updated_at}) do with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), {:ok, cursor} <- Search.update_search_cursor(search_cursor, %{ @@ -145,22 +152,27 @@ defmodule Algora.Github.Poller.Search do end end - defp process_ticket( - %{"updated_at" => updated_at, "body" => body, "html_url" => html_url} = ticket - ) do + defp process_ticket(%{"updatedAt" => updated_at, "url" => url} = ticket) do with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), - {:ok, [ticket_ref: ticket_ref], _, _, _, _} <- Parser.full_ticket_ref(html_url), - {:ok, commands} <- Command.parse(body) do + {:ok, [ticket_ref: ticket_ref], _, _, _, _} <- Parser.full_ticket_ref(url) do Logger.info("Latency: #{DateTime.diff(DateTime.utc_now(), updated_at, :second)}s") - Enum.reduce_while(commands, :ok, fn command, _acc -> + commands = + Enum.flat_map(ticket["comments"]["nodes"], fn comment -> + case Command.parse(comment["body"]) do + {:ok, [command | _]} -> [{comment, command}] + _ -> [] + end + end) + + Enum.reduce_while(commands, :ok, fn {comment, command}, _acc -> res = %{ - ticket: ticket, + comment: comment, command: Util.term_to_base64(command), ticket_ref: Util.term_to_base64(ticket_ref) } - |> Github.Poller.CommentConsumer.new() + |> Github.Poller.SearchConsumer.new() |> Oban.insert() case res do @@ -177,4 +189,46 @@ defmodule Algora.Github.Poller.Search do :ok end end + + defp search(q, opts \\ []) do + per_page = opts[:per_page] || 10 + since = opts[:since] || "2025-03-18T00:00:00Z" + + search_query = + if opts[:since] do + "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc updated:>#{opts[:since]}" + else + "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc" + end + + query = """ + query issues($search_query: String!) { + search(first: #{per_page}, type: ISSUE, query: $search_query) { + issueCount + pageInfo { + hasNextPage + } + nodes { + __typename + ... on Issue { + url + updatedAt + comments(first: 3, orderBy: {field: UPDATED_AT, direction: ASC}) { + nodes { + databaseId + author { + login + } + body + } + } + } + } + } + } + """ + + body = %{query: query, variables: %{search_query: search_query}} + Github.Client.fetch(Admin.token!(), "/graphql", "POST", body) + end end diff --git a/lib/algora/integrations/github/poller/search_consumer.ex b/lib/algora/integrations/github/poller/search_consumer.ex index a91756ae6..b0584770f 100644 --- a/lib/algora/integrations/github/poller/search_consumer.ex +++ b/lib/algora/integrations/github/poller/search_consumer.ex @@ -41,7 +41,10 @@ defmodule Algora.Github.Poller.SearchConsumer do end defp run_command({:bounty, args}, ticket_ref, comment) do - case Accounts.fetch_user_by(provider_id: to_string(comment["user"]["id"])) do + case Accounts.fetch_user_by( + provider: "github", + provider_login: to_string(comment["author"]["login"]) + ) do {:ok, user} -> Bounties.create_bounty( %{ @@ -54,7 +57,7 @@ defmodule Algora.Github.Poller.SearchConsumer do number: ticket_ref[:number] } }, - command_id: comment["id"], + command_id: comment["databaseId"], command_source: :comment ) diff --git a/lib/algora/integrations/github/poller/supervisor.ex b/lib/algora/integrations/github/poller/supervisor.ex index e4dc3f4d5..340eca5a6 100644 --- a/lib/algora/integrations/github/poller/supervisor.ex +++ b/lib/algora/integrations/github/poller/supervisor.ex @@ -2,8 +2,8 @@ defmodule Algora.Github.Poller.Supervisor do @moduledoc false use DynamicSupervisor - alias Algora.Comments - alias Algora.Github.Poller.Comments, as: CommentsPoller + alias Algora.Search + alias Algora.Github.Poller.Search, as: SearchPoller alias Algora.Github.TokenPool alias Algora.Workspace @@ -20,9 +20,9 @@ defmodule Algora.Github.Poller.Supervisor do end def start_children do - Comments.list_cursors() + Search.list_cursors() |> Task.async_stream( - fn cursor -> add_repo(cursor.repo_owner, cursor.repo_name) end, + fn cursor -> add_provider(cursor.provider) end, max_concurrency: 100, ordered: false ) @@ -31,61 +31,54 @@ defmodule Algora.Github.Poller.Supervisor do :ok end - def add_repo(owner, name, opts \\ []) do - token = TokenPool.get_token() - - case Workspace.ensure_repository(token, owner, name) do - {:ok, _repository} -> - spec = {CommentsPoller, [repo_owner: owner, repo_name: name] ++ opts} - DynamicSupervisor.start_child(__MODULE__, spec) - - error -> - error - end + def add_provider(provider, opts \\ []) do + DynamicSupervisor.start_child(__MODULE__, {SearchPoller, [provider: provider] ++ opts}) end - def terminate_child(owner, name) do - case find_child(owner, name) do + def terminate_child(provider) do + case find_child(provider) do {_id, pid, _type, _modules} -> DynamicSupervisor.terminate_child(__MODULE__, pid) nil -> {:error, :not_found} end end - def remove_repo(owner, name) do - with :ok <- terminate_child(owner, name), - {:ok, _cursor} <- Comments.delete_comment_cursor("github", owner, name) do + def remove_provider(provider) do + with :ok <- terminate_child(provider), + {:ok, _cursor} <- Search.delete_search_cursor(provider) do :ok end end - def find_child(owner, name) do - Enum.find(which_children(), fn {_, pid, _, _} -> GenServer.call(pid, :get_repo_info) == {owner, name} end) + def find_child(provider) do + Enum.find(which_children(), fn {_, pid, _, _} -> + GenServer.call(pid, :get_provider) == provider + end) end - def pause(owner, name) do - owner - |> find_child(name) + def pause(provider) do + provider + |> find_child() |> case do - {_, pid, _, _} -> CommentsPoller.pause(pid) + {_, pid, _, _} -> SearchPoller.pause(pid) nil -> {:error, :not_found} end end - def resume(owner, name) do - owner - |> find_child(name) + def resume(provider) do + provider + |> find_child() |> case do - {_, pid, _, _} -> CommentsPoller.resume(pid) + {_, pid, _, _} -> SearchPoller.resume(pid) nil -> {:error, :not_found} end end def pause_all do - Enum.each(which_children(), fn {_, pid, _, _} -> CommentsPoller.pause(pid) end) + Enum.each(which_children(), fn {_, pid, _, _} -> SearchPoller.pause(pid) end) end def resume_all do - Enum.each(which_children(), fn {_, pid, _, _} -> CommentsPoller.resume(pid) end) + Enum.each(which_children(), fn {_, pid, _, _} -> SearchPoller.resume(pid) end) end def which_children do From 00e91bbe2059cb480d0d2833331a6671f8b12fe3 Mon Sep 17 00:00:00 2001 From: zafer Date: Wed, 19 Mar 2025 18:54:52 +0200 Subject: [PATCH 3/5] chore: update GitHub integration and formatting - Enabled plugins in the formatter configuration for improved code formatting. - Cleaned up formatting in `mix.exs` for better readability. - Added `GITHUB_BOT_HANDLE` to GitHub configuration in both `dev.exs` and `runtime.exs`. - Refactored `Github.Client` and `Github.Poller.Search` for cleaner code and improved readability. - Adjusted `Search` module to ensure proper aliasing and organization. - Minor formatting adjustments in `HomeLive` for consistency. --- .formatter.exs | 2 +- config/dev.exs | 1 + config/runtime.exs | 5 ++- lib/algora/integrations/github/client.ex | 7 +-- lib/algora/integrations/github/github.ex | 1 + .../integrations/github/poller/search.ex | 43 +++++++++++-------- .../github/poller/search_consumer.ex | 7 +-- .../integrations/github/poller/supervisor.ex | 2 +- lib/algora/search/search.ex | 2 +- lib/algora_web/live/home_live.ex | 6 +-- mix.exs | 3 +- 11 files changed, 39 insertions(+), 40 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index 7369d4e36..d487aae7d 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,6 +1,6 @@ [ import_deps: [:ecto, :ecto_sql, :phoenix], subdirectories: ["priv/*/migrations", "config"], - # plugins: [Phoenix.LiveView.HTMLFormatter, Styler], + plugins: [Phoenix.LiveView.HTMLFormatter, Styler], inputs: ["*.{heex,ex,exs}", "{scripts,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] ] diff --git a/config/dev.exs b/config/dev.exs index db732c8c5..39329d743 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -9,6 +9,7 @@ config :algora, :github, private_key: System.get_env("GITHUB_PRIVATE_KEY"), pat: System.get_env("GITHUB_PAT"), pat_enabled: System.get_env("GITHUB_PAT_ENABLED", "false") == "true", + bot_handle: System.get_env("GITHUB_BOT_HANDLE"), oauth_state_ttl: String.to_integer(System.get_env("GITHUB_OAUTH_STATE_TTL", "600")), oauth_state_salt: System.get_env("GITHUB_OAUTH_STATE_SALT", "github-oauth-state") diff --git a/config/runtime.exs b/config/runtime.exs index b4b3dddb9..8c68cbae1 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -28,8 +28,9 @@ if config_env() == :prod do app_id: System.fetch_env!("GITHUB_APP_ID"), webhook_secret: System.fetch_env!("GITHUB_WEBHOOK_SECRET"), private_key: System.fetch_env!("GITHUB_PRIVATE_KEY"), - pat: System.fetch_env!("GITHUB_PAT"), - pat_enabled: System.get_env("GITHUB_PAT_ENABLED", "true") == "true", + pat: System.get_env("GITHUB_PAT"), + pat_enabled: System.get_env("GITHUB_PAT_ENABLED", "false") == "true", + bot_handle: System.get_env("GITHUB_BOT_HANDLE"), oauth_state_ttl: String.to_integer(System.get_env("GITHUB_OAUTH_STATE_TTL", "600")), oauth_state_salt: System.fetch_env!("GITHUB_OAUTH_STATE_SALT") diff --git a/lib/algora/integrations/github/client.ex b/lib/algora/integrations/github/client.ex index 477eea116..240b4d0e1 100644 --- a/lib/algora/integrations/github/client.ex +++ b/lib/algora/integrations/github/client.ex @@ -34,9 +34,7 @@ defmodule Algora.Github.Client do {:ok, %Finch.Response{status: status, headers: headers}} when status in [301, 302, 307] -> case List.keyfind(headers, "location", 0) do {"location", location} -> - request_with_follow_redirects( - Finch.build(request.method, location, request.headers, request.body) - ) + request_with_follow_redirects(Finch.build(request.method, location, request.headers, request.body)) nil -> {:error, "Redirect response missing location header"} @@ -101,8 +99,7 @@ defmodule Algora.Github.Client do def fetch(access_token, url, method \\ "GET", body \\ nil) - def fetch(access_token, "https://api.github.com" <> path, method, body), - do: fetch(access_token, path, method, body) + def fetch(access_token, "https://api.github.com" <> path, method, body), do: fetch(access_token, path, method, body) def fetch(access_token, path, method, body) do http( diff --git a/lib/algora/integrations/github/github.ex b/lib/algora/integrations/github/github.ex index f9ff91358..394d38bf5 100644 --- a/lib/algora/integrations/github/github.ex +++ b/lib/algora/integrations/github/github.ex @@ -14,6 +14,7 @@ defmodule Algora.Github do def private_key, do: [:github, :private_key] |> Algora.config() |> String.replace("\\n", "\n") def pat, do: Algora.config([:github, :pat]) def pat_enabled, do: Algora.config([:github, :pat_enabled]) + def bot_handle, do: Algora.config([:github, :bot_handle]) def install_url_base, do: "https://github.com/apps/#{app_handle()}/installations" def install_url_new, do: "#{install_url_base()}/new" diff --git a/lib/algora/integrations/github/poller/search.ex b/lib/algora/integrations/github/poller/search.ex index c030acc39..c1dfb02d0 100644 --- a/lib/algora/integrations/github/poller/search.ex +++ b/lib/algora/integrations/github/poller/search.ex @@ -4,12 +4,12 @@ defmodule Algora.Github.Poller.Search do import Ecto.Query, warn: false - alias Algora.Search + alias Algora.Admin alias Algora.Github alias Algora.Github.Command alias Algora.Parser - alias Algora.Admin alias Algora.Repo + alias Algora.Search alias Algora.Util require Logger @@ -129,7 +129,7 @@ defmodule Algora.Github.Poller.Search do end end - defp get_or_create_cursor() do + defp get_or_create_cursor do case Search.get_search_cursor("github") do nil -> Search.create_search_cursor(%{provider: "github", timestamp: DateTime.utc_now()}) @@ -152,20 +152,27 @@ defmodule Algora.Github.Poller.Search do end end - defp process_ticket(%{"updatedAt" => updated_at, "url" => url} = ticket) do + defp process_ticket(%{"updatedAt" => updated_at, "url" => url} = ticket, state) do + dbg(ticket) + with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), {:ok, [ticket_ref: ticket_ref], _, _, _, _} <- Parser.full_ticket_ref(url) do Logger.info("Latency: #{DateTime.diff(DateTime.utc_now(), updated_at, :second)}s") - commands = - Enum.flat_map(ticket["comments"]["nodes"], fn comment -> - case Command.parse(comment["body"]) do - {:ok, [command | _]} -> [{comment, command}] - _ -> [] - end - end) + ticket["comments"]["nodes"] + |> Enum.reject(fn comment -> + comment["author"]["login"] == Github.bot_handle() or + DateTime.before?(updated_at, state.cursor.timestamp) + end) + |> Enum.flat_map(fn comment -> + dbg(comment) - Enum.reduce_while(commands, :ok, fn {comment, command}, _acc -> + case Command.parse(comment["body"]) do + {:ok, [command | _]} -> [{comment, command}] + _ -> [] + end + end) + |> Enum.reduce_while(:ok, fn {comment, command}, _acc -> res = %{ comment: comment, @@ -182,17 +189,15 @@ defmodule Algora.Github.Poller.Search do end) else {:error, reason} -> - Logger.error( - "Failed to parse commands from ticket: #{inspect(ticket)}. Reason: #{inspect(reason)}" - ) + Logger.error("Failed to parse commands from ticket: #{inspect(ticket)}. Reason: #{inspect(reason)}") :ok end end - defp search(q, opts \\ []) do + def search(q, opts \\ []) do per_page = opts[:per_page] || 10 - since = opts[:since] || "2025-03-18T00:00:00Z" + since = opts[:since] search_query = if opts[:since] do @@ -201,6 +206,8 @@ defmodule Algora.Github.Poller.Search do "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc" end + dbg(search_query) + query = """ query issues($search_query: String!) { search(first: #{per_page}, type: ISSUE, query: $search_query) { @@ -213,7 +220,7 @@ defmodule Algora.Github.Poller.Search do ... on Issue { url updatedAt - comments(first: 3, orderBy: {field: UPDATED_AT, direction: ASC}) { + comments(last: 3, orderBy: {field: UPDATED_AT, direction: DESC}) { nodes { databaseId author { diff --git a/lib/algora/integrations/github/poller/search_consumer.ex b/lib/algora/integrations/github/poller/search_consumer.ex index b0584770f..8e2bc75ee 100644 --- a/lib/algora/integrations/github/poller/search_consumer.ex +++ b/lib/algora/integrations/github/poller/search_consumer.ex @@ -10,12 +10,7 @@ defmodule Algora.Github.Poller.SearchConsumer do @impl Oban.Worker def perform(%Oban.Job{ - args: - %{ - "comment" => comment, - "command" => encoded_command, - "ticket_ref" => encoded_ticket_ref - } = _args + args: %{"comment" => comment, "command" => encoded_command, "ticket_ref" => encoded_ticket_ref} = _args }) do command = Util.base64_to_term!(encoded_command) ticket_ref = Util.base64_to_term!(encoded_ticket_ref) diff --git a/lib/algora/integrations/github/poller/supervisor.ex b/lib/algora/integrations/github/poller/supervisor.ex index 340eca5a6..ec1436acc 100644 --- a/lib/algora/integrations/github/poller/supervisor.ex +++ b/lib/algora/integrations/github/poller/supervisor.ex @@ -2,9 +2,9 @@ defmodule Algora.Github.Poller.Supervisor do @moduledoc false use DynamicSupervisor - alias Algora.Search alias Algora.Github.Poller.Search, as: SearchPoller alias Algora.Github.TokenPool + alias Algora.Search alias Algora.Workspace require Logger diff --git a/lib/algora/search/search.ex b/lib/algora/search/search.ex index 4063db5fa..663ac6e29 100644 --- a/lib/algora/search/search.ex +++ b/lib/algora/search/search.ex @@ -2,8 +2,8 @@ defmodule Algora.Search do @moduledoc false import Ecto.Query, warn: false - alias Algora.Search.SearchCursor alias Algora.Repo + alias Algora.Search.SearchCursor def get_search_cursor(provider) do Repo.get_by(SearchCursor, provider: provider) diff --git a/lib/algora_web/live/home_live.ex b/lib/algora_web/live/home_live.ex index cc28d3536..6545b7a92 100644 --- a/lib/algora_web/live/home_live.ex +++ b/lib/algora_web/live/home_live.ex @@ -862,8 +862,7 @@ defmodule AlgoraWeb.HomeLive do |> redirect(to: ~p"/")} {:error, :already_exists} -> - {:noreply, - put_flash(socket, :warning, "You have already created a bounty for this ticket")} + {:noreply, put_flash(socket, :warning, "You have already created a bounty for this ticket")} {:error, _reason} -> {:noreply, put_flash(socket, :error, "Something went wrong")} @@ -1099,8 +1098,7 @@ defmodule AlgoraWeb.HomeLive do """ end - defp format_money(money), - do: money |> Money.round(currency_digits: 0) |> Money.to_string!(no_fraction_if_integer: true) + defp format_money(money), do: money |> Money.round(currency_digits: 0) |> Money.to_string!(no_fraction_if_integer: true) defp format_number(number), do: Number.Delimit.number_to_delimited(number, precision: 0) diff --git a/mix.exs b/mix.exs index b3454783b..a8732bd1d 100644 --- a/mix.exs +++ b/mix.exs @@ -56,8 +56,7 @@ defmodule Algora.MixProject do {:floki, ">= 0.30.0"}, {:phoenix_live_dashboard, "~> 0.8.3"}, {:tailwind, "~> 0.2", runtime: Mix.env() == :dev}, - {:tabler_icons, - github: "algora-io/icons", sparse: "icons", app: false, compile: false, depth: 1}, + {:tabler_icons, github: "algora-io/icons", sparse: "icons", app: false, compile: false, depth: 1}, {:swoosh, "~> 1.5"}, {:finch, "~> 0.13"}, {:httpoison, "~> 2.2"}, From 446b658646729da8608c4360288f00453d06b086 Mon Sep 17 00:00:00 2001 From: zafer Date: Wed, 19 Mar 2025 19:04:20 +0200 Subject: [PATCH 4/5] fix --- .../integrations/github/poller/search.ex | 29 +++++++++++++------ .../integrations/github/poller/supervisor.ex | 4 +-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/lib/algora/integrations/github/poller/search.ex b/lib/algora/integrations/github/poller/search.ex index c1dfb02d0..a9825a6d8 100644 --- a/lib/algora/integrations/github/poller/search.ex +++ b/lib/algora/integrations/github/poller/search.ex @@ -91,7 +91,7 @@ defmodule Algora.Github.Poller.Search do def poll(state) do with {:ok, tickets} <- fetch_tickets(state), if(length(tickets) > 0, do: Logger.debug("Processing #{length(tickets)} tickets")), - {:ok, updated_cursor} <- process_batch(tickets, state.cursor) do + {:ok, updated_cursor} <- process_batch(tickets, state) do {:ok, %{state | cursor: updated_cursor}} else {:error, reason} -> @@ -100,19 +100,19 @@ defmodule Algora.Github.Poller.Search do end end - defp process_batch([], search_cursor), do: {:ok, search_cursor} + defp process_batch([], state), do: {:ok, state.cursor} - defp process_batch(tickets, search_cursor) do + defp process_batch(tickets, state) do Repo.transact(fn -> - with :ok <- process_tickets(tickets) do - update_last_polled(search_cursor, List.last(tickets)) + with :ok <- process_tickets(tickets, state) do + update_last_polled(state.cursor, List.last(tickets)) end end) end - defp process_tickets(tickets) do + defp process_tickets(tickets, state) do Enum.reduce_while(tickets, :ok, fn ticket, _acc -> - case process_ticket(ticket) do + case process_ticket(ticket, state) do {:ok, _} -> {:cont, :ok} error -> {:halt, error} end @@ -161,8 +161,18 @@ defmodule Algora.Github.Poller.Search do ticket["comments"]["nodes"] |> Enum.reject(fn comment -> - comment["author"]["login"] == Github.bot_handle() or - DateTime.before?(updated_at, state.cursor.timestamp) + already_processed? = + case DateTime.from_iso8601(comment["updatedAt"]) do + {:ok, comment_updated_at, _} -> + DateTime.before?(comment_updated_at, state.cursor.timestamp) + + {:error, _} -> + true + end + + bot? = comment["author"]["login"] == Github.bot_handle() + + bot? or already_processed? end) |> Enum.flat_map(fn comment -> dbg(comment) @@ -222,6 +232,7 @@ defmodule Algora.Github.Poller.Search do updatedAt comments(last: 3, orderBy: {field: UPDATED_AT, direction: DESC}) { nodes { + updatedAt databaseId author { login diff --git a/lib/algora/integrations/github/poller/supervisor.ex b/lib/algora/integrations/github/poller/supervisor.ex index ec1436acc..63b2079cd 100644 --- a/lib/algora/integrations/github/poller/supervisor.ex +++ b/lib/algora/integrations/github/poller/supervisor.ex @@ -31,7 +31,7 @@ defmodule Algora.Github.Poller.Supervisor do :ok end - def add_provider(provider, opts \\ []) do + def add_provider(provider \\ "github", opts \\ []) do DynamicSupervisor.start_child(__MODULE__, {SearchPoller, [provider: provider] ++ opts}) end @@ -42,7 +42,7 @@ defmodule Algora.Github.Poller.Supervisor do end end - def remove_provider(provider) do + def remove_provider(provider \\ "github") do with :ok <- terminate_child(provider), {:ok, _cursor} <- Search.delete_search_cursor(provider) do :ok From de445e0ec3b1055d72e6ffd2d96541a61b54ac54 Mon Sep 17 00:00:00 2001 From: zafer Date: Wed, 19 Mar 2025 19:54:37 +0200 Subject: [PATCH 5/5] refactor: improve code readability and structure - Updated `NotifyBounty` to use structured ticket reference for better clarity. - Enhanced `SearchConsumer` to implement a strategy for command responses based on existing entries. - Cleaned up formatting and improved readability across multiple modules, including `GithubController` and `Workspace`. - Removed unnecessary debug statements and added error logging for better traceability. --- lib/algora/bounties/jobs/notify_bounty.ex | 45 +++++++++------- .../integrations/github/poller/search.ex | 6 --- .../github/poller/search_consumer.ex | 15 +++++- lib/algora/workspace/workspace.ex | 20 +++++-- .../controllers/webhooks/github_controller.ex | 54 ++++++++++++++----- 5 files changed, 97 insertions(+), 43 deletions(-) diff --git a/lib/algora/bounties/jobs/notify_bounty.ex b/lib/algora/bounties/jobs/notify_bounty.ex index 9b531d576..48774232a 100644 --- a/lib/algora/bounties/jobs/notify_bounty.ex +++ b/lib/algora/bounties/jobs/notify_bounty.ex @@ -21,34 +21,39 @@ defmodule Algora.Bounties.Jobs.NotifyBounty do "command_source" => command_source } }) do + ticket_ref = %{ + owner: ticket_ref["owner"], + repo: ticket_ref["repo"], + number: ticket_ref["number"] + } + body = """ 💎 **#{owner_login}** is offering a **#{amount}** bounty for this issue - 👉 Got a pull request resolving this? Claim the bounty by commenting `/claim ##{ticket_ref["number"]}` in your PR and joining #{AlgoraWeb.Endpoint.host()} + 👉 Got a pull request resolving this? Claim the bounty by commenting `/claim ##{ticket_ref.number}` in your PR and joining #{AlgoraWeb.Endpoint.host()} """ if Github.pat_enabled() do - with {:ok, comment} <- - Github.create_issue_comment( + with {:ok, ticket} <- + Workspace.ensure_ticket( Github.pat(), - ticket_ref["owner"], - ticket_ref["repo"], - ticket_ref["number"], - body - ), - {:ok, ticket} <- - Workspace.ensure_ticket(Github.pat(), ticket_ref["owner"], ticket_ref["repo"], ticket_ref["number"]) do - # TODO: update existing command response if it exists - Workspace.create_command_response(%{ - comment: comment, - command_source: command_source, + ticket_ref.owner, + ticket_ref.repo, + ticket_ref.number + ) do + Workspace.ensure_command_response(%{ + token: Github.pat(), + ticket_ref: ticket_ref, command_id: command_id, - ticket_id: ticket.id + command_type: :bounty, + command_source: command_source, + ticket: ticket, + body: body }) end else Logger.info(""" - Github.create_issue_comment(Github.pat(), "#{ticket_ref["owner"]}", "#{ticket_ref["repo"]}", #{ticket_ref["number"]}, + Github.create_issue_comment(Github.pat(), "#{ticket_ref.owner}", "#{ticket_ref.repo}", #{ticket_ref.number}, \"\"\" #{body} \"\"\") @@ -73,9 +78,13 @@ defmodule Algora.Bounties.Jobs.NotifyBounty do } with {:ok, token} <- Github.get_installation_token(installation_id), - {:ok, ticket} <- Workspace.ensure_ticket(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number), + {:ok, ticket} <- + Workspace.ensure_ticket(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number), bounties when bounties != [] <- Bounties.list_bounties(ticket_id: ticket.id), - {:ok, _} <- Github.add_labels(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number, ["💎 Bounty"]) do + {:ok, _} <- + Github.add_labels(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number, [ + "💎 Bounty" + ]) do attempts = Bounties.list_attempts_for_ticket(ticket.id) claims = Bounties.list_claims([ticket.id]) diff --git a/lib/algora/integrations/github/poller/search.ex b/lib/algora/integrations/github/poller/search.ex index a9825a6d8..49a106930 100644 --- a/lib/algora/integrations/github/poller/search.ex +++ b/lib/algora/integrations/github/poller/search.ex @@ -153,8 +153,6 @@ defmodule Algora.Github.Poller.Search do end defp process_ticket(%{"updatedAt" => updated_at, "url" => url} = ticket, state) do - dbg(ticket) - with {:ok, updated_at, _} <- DateTime.from_iso8601(updated_at), {:ok, [ticket_ref: ticket_ref], _, _, _, _} <- Parser.full_ticket_ref(url) do Logger.info("Latency: #{DateTime.diff(DateTime.utc_now(), updated_at, :second)}s") @@ -175,8 +173,6 @@ defmodule Algora.Github.Poller.Search do bot? or already_processed? end) |> Enum.flat_map(fn comment -> - dbg(comment) - case Command.parse(comment["body"]) do {:ok, [command | _]} -> [{comment, command}] _ -> [] @@ -216,8 +212,6 @@ defmodule Algora.Github.Poller.Search do "#{q} in:comment is:issue repo:acme-incorporated/webapp sort:updated-asc" end - dbg(search_query) - query = """ query issues($search_query: String!) { search(first: #{per_page}, type: ISSUE, query: $search_query) { diff --git a/lib/algora/integrations/github/poller/search_consumer.ex b/lib/algora/integrations/github/poller/search_consumer.ex index 8e2bc75ee..b5dddcd99 100644 --- a/lib/algora/integrations/github/poller/search_consumer.ex +++ b/lib/algora/integrations/github/poller/search_consumer.ex @@ -4,7 +4,9 @@ defmodule Algora.Github.Poller.SearchConsumer do alias Algora.Accounts alias Algora.Bounties + alias Algora.Repo alias Algora.Util + alias Algora.Workspace.CommandResponse require Logger @@ -41,6 +43,16 @@ defmodule Algora.Github.Poller.SearchConsumer do provider_login: to_string(comment["author"]["login"]) ) do {:ok, user} -> + strategy = + case Repo.get_by(CommandResponse, + provider: "github", + provider_command_id: to_string(comment["databaseId"]), + command_source: :comment + ) do + nil -> :increase + _ -> :set + end + Bounties.create_bounty( %{ creator: user, @@ -53,7 +65,8 @@ defmodule Algora.Github.Poller.SearchConsumer do } }, command_id: comment["databaseId"], - command_source: :comment + command_source: :comment, + strategy: strategy ) {:error, _reason} = error -> diff --git a/lib/algora/workspace/workspace.ex b/lib/algora/workspace/workspace.ex index 43d5f9854..f43269f34 100644 --- a/lib/algora/workspace/workspace.ex +++ b/lib/algora/workspace/workspace.ex @@ -142,7 +142,8 @@ defmodule Algora.Workspace do needs_update? = Repository.has_default_og_image?(repository) || - (repository.og_image_updated_at && DateTime.before?(repository.og_image_updated_at, one_day_ago)) + (repository.og_image_updated_at && + DateTime.before?(repository.og_image_updated_at, one_day_ago)) if needs_update? do %{repository_id: repository.id} @@ -162,7 +163,9 @@ defmodule Algora.Workspace do else {:error, %Ecto.Changeset{ - errors: [provider: {_, [constraint: :unique, constraint_name: "repositories_provider_provider_id_index"]}] + errors: [ + provider: {_, [constraint: :unique, constraint_name: "repositories_provider_provider_id_index"]} + ] } = changeset} -> Repo.fetch_by(Repository, provider: "github", provider_id: changeset.changes.provider_id) @@ -220,6 +223,7 @@ defmodule Algora.Workspace do error -> Logger.error("#{owner}/#{repo} | failed to remap #{user.provider_login} -> #{github_user["login"]}") + error end end @@ -384,14 +388,21 @@ defmodule Algora.Workspace do {:error, reason} end - {:error, _reason} -> + {:error, reason} -> + Logger.error("Failed to refresh command response #{ticket.id}: #{inspect(reason)}") {:error, :command_response_not_found} end end defp post_response(token, ticket_ref, command_id, command_source, ticket, body) do with {:ok, comment} <- - Github.create_issue_comment(token, ticket_ref[:owner], ticket_ref[:repo], ticket_ref[:number], body) do + Github.create_issue_comment( + token, + ticket_ref[:owner], + ticket_ref[:repo], + ticket_ref[:number], + body + ) do create_command_response(%{ comment: comment, command_source: command_source, @@ -435,6 +446,7 @@ defmodule Algora.Workspace do {:error, reason} -> Logger.error("Failed to update command response #{command_response.id}: #{inspect(reason)}") + {:ok, command_response} end end diff --git a/lib/algora_web/controllers/webhooks/github_controller.ex b/lib/algora_web/controllers/webhooks/github_controller.ex index 40a8b88b0..e1d2880cd 100644 --- a/lib/algora_web/controllers/webhooks/github_controller.ex +++ b/lib/algora_web/controllers/webhooks/github_controller.ex @@ -64,7 +64,12 @@ defmodule AlgoraWeb.Webhooks.GithubController do with {:ok, token} <- Github.get_installation_token(payload["installation"]["id"]), {:ok, %{"permission" => permission}} <- - Github.get_repository_permissions(token, repo["owner"]["login"], repo["name"], author["login"]) do + Github.get_repository_permissions( + token, + repo["owner"]["login"], + repo["name"], + author["login"] + ) do case permission do "admin" -> {:ok, :admin} "write" -> {:ok, :mod} @@ -77,7 +82,11 @@ defmodule AlgoraWeb.Webhooks.GithubController do installation_id = payload["installation"]["id"] with {:ok, token} <- Github.get_installation_token(installation_id), - {:ok, installation} <- Repo.fetch_by(Installation, provider: "github", provider_id: to_string(installation_id)), + {:ok, installation} <- + Repo.fetch_by(Installation, + provider: "github", + provider_id: to_string(installation_id) + ), {:ok, user} <- Workspace.ensure_user(token, author["login"]) do member = Repo.one( @@ -166,7 +175,10 @@ defmodule AlgoraWeb.Webhooks.GithubController do primary_claim = List.first(claims) installation = - Repo.get_by(Installation, provider: "github", provider_id: to_string(payload["installation"]["id"])) + Repo.get_by(Installation, + provider: "github", + provider_id: to_string(payload["installation"]["id"]) + ) bounties = Repo.all( @@ -220,6 +232,7 @@ defmodule AlgoraWeb.Webhooks.GithubController do %{idempotency_key: idempotency_key} ) do Logger.info("Autopay successful (#{autopayable_bounty.owner.name} - #{autopayable_bounty.amount}).") + :ok else {:error, reason} -> @@ -314,9 +327,6 @@ defmodule AlgoraWeb.Webhooks.GithubController do ticket_number = get_github_ticket(webhook)["number"] - # TODO: perform compensating action if needed - # ❌ comment1.created (:set) -> comment2.created (:increase) -> comment2.edited (:increase) - # ✅ comment1.created (:set) -> comment2.created (:increase) -> comment2.edited (:decrease + :increase) strategy = case Repo.get_by(CommandResponse, provider: "github", @@ -331,7 +341,10 @@ defmodule AlgoraWeb.Webhooks.GithubController do with {:ok, role} when role in [:admin, :mod] <- authorize_user(webhook), {:ok, token} <- Github.get_installation_token(payload["installation"]["id"]), {:ok, installation} <- - Workspace.fetch_installation_by(provider: "github", provider_id: to_string(payload["installation"]["id"])), + Workspace.fetch_installation_by( + provider: "github", + provider_id: to_string(payload["installation"]["id"]) + ), {:ok, owner} <- Accounts.fetch_user_by(id: installation.connected_user_id), {:ok, creator} <- Workspace.ensure_user(token, author["login"]) do Bounties.create_bounty( @@ -372,7 +385,10 @@ defmodule AlgoraWeb.Webhooks.GithubController do case authorize_user(webhook) do {:ok, role} when role in [:admin, :mod] -> installation = - Repo.get_by(Installation, provider: "github", provider_id: to_string(payload["installation"]["id"])) + Repo.get_by(Installation, + provider: "github", + provider_id: to_string(payload["installation"]["id"]) + ) customer = Repo.one( @@ -386,7 +402,8 @@ defmodule AlgoraWeb.Webhooks.GithubController do {:ok, token} = Github.get_installation_token(payload["installation"]["id"]) - {:ok, ticket} = Workspace.ensure_ticket(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number) + {:ok, ticket} = + Workspace.ensure_ticket(token, ticket_ref.owner, ticket_ref.repo, ticket_ref.number) autopay_cooldown_expired? = fn -> from(t in Tip, @@ -399,8 +416,11 @@ defmodule AlgoraWeb.Webhooks.GithubController do ) |> Repo.one() |> case do - nil -> true - tip -> DateTime.diff(DateTime.utc_now(), tip.inserted_at, :millisecond) > :timer.hours(1) + nil -> + true + + tip -> + DateTime.diff(DateTime.utc_now(), tip.inserted_at, :millisecond) > :timer.hours(1) end end @@ -417,7 +437,8 @@ defmodule AlgoraWeb.Webhooks.GithubController do autopay_result = if autopayable? do with {:ok, owner} <- Accounts.fetch_user_by(id: installation.connected_user_id), - {:ok, creator} <- Workspace.ensure_user(token, payload["repository"]["owner"]["login"]), + {:ok, creator} <- + Workspace.ensure_user(token, payload["repository"]["owner"]["login"]), {:ok, recipient} <- Workspace.ensure_user(token, recipient), {:ok, tip} <- Bounties.do_create_tip( @@ -446,6 +467,7 @@ defmodule AlgoraWeb.Webhooks.GithubController do %{idempotency_key: idempotency_key} ) do Logger.info("Autopay successful (#{payload["repository"]["full_name"]}##{ticket_ref.number} - #{amount}).") + {:ok, tip} else {:error, reason} -> @@ -572,8 +594,11 @@ defmodule AlgoraWeb.Webhooks.GithubController do |> Enum.reject(&is_nil/1) # keep only the first claim command if multiple claims are present |> Enum.reduce([], fn - {:claim, _} = claim, acc -> if Enum.any?(acc, &match?({:claim, _}, &1)), do: acc, else: [claim | acc] - command, acc -> [command | acc] + {:claim, _} = claim, acc -> + if Enum.any?(acc, &match?({:claim, _}, &1)), do: acc, else: [claim | acc] + + command, acc -> + [command | acc] end) |> Enum.reverse()} @@ -599,6 +624,7 @@ defmodule AlgoraWeb.Webhooks.GithubController do error -> Logger.error("Command execution failed for #{event_action}(#{hook_id}): #{inspect(command)}: #{inspect(error)}") + {:halt, error} end end)