diff --git a/config/test.exs b/config/test.exs index e4fe03f50..0c602e528 100644 --- a/config/test.exs +++ b/config/test.exs @@ -23,3 +23,4 @@ config :accent, Accent.Mailer, adapter: Bamboo.TestAdapter config :accent, hook_broadcaster: Accent.Hook.BroadcasterMock +config :accent, hook_github_file_server: Accent.Hook.Consumers.GitHub.FileServerMock diff --git a/lib/accent.ex b/lib/accent.ex index ae76cb9ba..ee5bb4437 100644 --- a/lib/accent.ex +++ b/lib/accent.ex @@ -11,6 +11,8 @@ defmodule Accent do supervisor(Accent.Endpoint, []), # Start the Ecto repository worker(Accent.Repo, []), + worker(Accent.Hook.Producers.GitHub, []), + worker(Accent.Hook.Consumers.GitHub, []), worker(Accent.Hook.Producers.Email, []), worker(Accent.Hook.Consumers.Email, mailer: Accent.Mailer), worker(Accent.Hook.Producers.Websocket, []), diff --git a/lib/accent/auth/role_abilities.ex b/lib/accent/auth/role_abilities.ex index 74ce0e2be..6014c7165 100644 --- a/lib/accent/auth/role_abilities.ex +++ b/lib/accent/auth/role_abilities.ex @@ -41,6 +41,7 @@ defmodule Accent.RoleAbilities do peek_merge merge sync + hook_update )a ++ @any_actions @developer_actions ~w( diff --git a/lib/accent/schemas/integration.ex b/lib/accent/schemas/integration.ex index 6b361781e..75db99596 100644 --- a/lib/accent/schemas/integration.ex +++ b/lib/accent/schemas/integration.ex @@ -7,6 +7,9 @@ defmodule Accent.Integration do embeds_one(:data, IntegrationData, on_replace: :update) do field(:url) + field(:repository) + field(:token) + field(:default_ref, :string, default: "master") end belongs_to(:project, Accent.Project) diff --git a/lib/accent/schemas/project.ex b/lib/accent/schemas/project.ex index 436867d72..7c7065853 100644 --- a/lib/accent/schemas/project.ex +++ b/lib/accent/schemas/project.ex @@ -9,6 +9,7 @@ defmodule Accent.Project do has_many(:integrations, Accent.Integration) has_many(:revisions, Accent.Revision) + has_many(:target_revisions, Accent.Revision, where: [master: false]) has_many(:versions, Accent.Version) has_many(:operations, Accent.Operation) has_many(:collaborators, Accent.Collaborator) diff --git a/lib/accent/scopes/integration.ex b/lib/accent/scopes/integration.ex new file mode 100644 index 000000000..6ac2c5d4d --- /dev/null +++ b/lib/accent/scopes/integration.ex @@ -0,0 +1,36 @@ +defmodule Accent.Scopes.Integration do + import Ecto.Query, only: [from: 2] + + @doc """ + ## Examples + + iex> Accent.Scopes.Integration.from_project(Accent.Integration, "test") + #Ecto.Query + """ + @spec from_project(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t() + def from_project(query, project_id) do + from(query, where: [project_id: ^project_id]) + end + + @doc """ + ## Examples + + iex> Accent.Scopes.Integration.from_service(Accent.Integration, "test") + #Ecto.Query + """ + @spec from_service(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t() + def from_service(query, service) do + from(query, where: [service: ^service]) + end + + @doc """ + ## Examples + + iex> Accent.Scopes.Integration.from_data_repository(Accent.Integration, "test") + #Ecto.Query>'repository' = ?\", i0.data, ^\"test\")> + """ + @spec from_data_repository(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t() + def from_data_repository(query, repository) do + from(i in query, where: fragment("?->>'repository' = ?", i.data, ^repository)) + end +end diff --git a/lib/graphql/resolvers/collaborator.ex b/lib/graphql/resolvers/collaborator.ex index 0e64488b4..cf4dc2850 100644 --- a/lib/graphql/resolvers/collaborator.ex +++ b/lib/graphql/resolvers/collaborator.ex @@ -22,7 +22,7 @@ defmodule Accent.GraphQL.Resolvers.Collaborator do case CollaboratorCreator.create(params) do {:ok, collaborator} -> - Accent.Hook.fanout(%Hook.Context{ + Accent.Hook.notify(%Hook.Context{ event: "create_collaborator", project: project, user: info.context[:conn].assigns[:current_user], diff --git a/lib/graphql/resolvers/comment.ex b/lib/graphql/resolvers/comment.ex index 690daa594..3e5408844 100644 --- a/lib/graphql/resolvers/comment.ex +++ b/lib/graphql/resolvers/comment.ex @@ -27,7 +27,7 @@ defmodule Accent.GraphQL.Resolvers.Comment do {:ok, comment} -> comment = Repo.preload(comment, [:user, translation: [revision: :project]]) - Accent.Hook.fanout(%Hook.Context{ + Accent.Hook.notify(%Hook.Context{ event: "create_comment", project: comment.translation.revision.project, user: info.context[:conn].assigns[:current_user], diff --git a/lib/hook/broadcaster.ex b/lib/hook/broadcaster.ex index 943ed4712..cd9f72369 100644 --- a/lib/hook/broadcaster.ex +++ b/lib/hook/broadcaster.ex @@ -1,16 +1,22 @@ defmodule Accent.Hook.Broadcaster do - @producers [ + @notifiers [ Accent.Hook.Producers.Email, Accent.Hook.Producers.Websocket, Accent.Hook.Producers.Slack ] - @callback fanout(Accent.Hook.Context.t()) :: no_return() + @callback notify(Accent.Hook.Context.t()) :: no_return() + @callback external_document_update(:github, Accent.Hook.Context.t()) :: no_return() + @timeout 10_000 - def fanout(context = %Accent.Hook.Context{}) do - for producer <- @producers do + def notify(context = %Accent.Hook.Context{}) do + for producer <- @notifiers do GenStage.call(producer, {:notify, context}, @timeout) end end + + def external_document_update(:github, context = %Accent.Hook.Context{}) do + GenStage.call(Accent.Hook.Producers.GitHub, {:external_document_update, context}, @timeout) + end end diff --git a/lib/hook/consumers/github.ex b/lib/hook/consumers/github.ex new file mode 100644 index 000000000..c52d025e5 --- /dev/null +++ b/lib/hook/consumers/github.ex @@ -0,0 +1,140 @@ +defmodule Accent.Hook.Consumers.GitHub do + @moduledoc """ + From a project’s integration association and a webhook event from GitHub, + sync and add translation as if the operations were made manually by a user. + + """ + use Accent.Hook.EventConsumer, subscribe_to: [Accent.Hook.Producers.GitHub] + + alias Accent.{Document, Repo, Version} + alias Accent.Hook.Consumers.GitHub.AddTranslations + alias Accent.Hook.Consumers.GitHub.Sync + alias Accent.Hook.Context + alias Accent.Plugs.MovementContextParser + alias Accent.Scopes.Document, as: DocumentScope + alias Accent.Scopes.Version, as: VersionScope + + def handle_events(events, _from, state) do + Enum.each(events, &handle_event/1) + + {:noreply, [], state} + end + + defp handle_event(%Context{user: user, project: project, payload: payload}) do + payload[:ref] + |> ref_to_version(payload[:default_ref], project) + |> sync_and_add_translations(project, user, payload) + end + + defp sync_and_add_translations({:ok, version}, project, user, payload) do + repo = payload[:repository] + token = payload[:token] + + ref = (version && version.tag) || payload[:default_ref] + configs = fetch_config(repo, token, ref) + trees = fetch_trees(repo, token, ref) + + revisions = + project + |> Ecto.assoc(:target_revisions) + |> Repo.all() + |> Repo.preload(:language) + + Sync.persist(trees, configs, project, user, payload, version) + + Enum.each(revisions, fn revision -> + AddTranslations.persist(trees, configs, project, user, revision, payload, version) + end) + end + + defp sync_and_add_translations(_, _, _, _), do: :ok + + def filter_by_patterns(patterns, files) do + Enum.group_by(files, fn file -> Enum.find(patterns, &ExMinimatch.match(&1["matcher"], file["path"])) end) + end + + def movement_document(project, path) do + path = + path + |> Path.basename() + |> MovementContextParser.extract_path_from_filename() + + Document + |> DocumentScope.from_path(path) + |> DocumentScope.from_project(project.id) + |> Repo.one() + |> Kernel.||(%Document{project_id: project.id, path: path}) + end + + def fetch_content(path, token) do + with {:ok, %{body: %{"content" => content}}} <- file_server().get(path, headers(token)), + decoded_contents <- + content + |> String.split("\n") + |> Enum.reject(&(&1 === "")) + |> Enum.map(&Base.decode64/1), + true <- Enum.all?(decoded_contents, &match?({:ok, _}, &1)), + decoded_content <- + decoded_contents + |> Enum.map(&elem(&1, 1)) + |> Enum.join("") do + {:ok, decoded_content} + else + _ -> {:ok, nil} + end + end + + def ref_to_version(ref, default_ref, project) do + default_version(ref, default_ref) || version_from_ref(ref, project) + end + + defp default_version(ref, default_ref) do + case Regex.named_captures(~r/refs\/heads\/(?.+)/, ref) do + %{"branch" => ^default_ref} -> {:ok, nil} + _ -> nil + end + end + + defp version_from_ref(ref, project) do + with %{"tag" => tag} <- Regex.named_captures(~r/refs\/tags\/(?.+)/, ref), + version = %Version{} <- + Version + |> VersionScope.from_project(project.id) + |> VersionScope.from_tag(tag) + |> Repo.one() do + {:ok, version} + else + _ -> + nil + end + end + + defp fetch_config(repo, token, ref) do + with path <- Path.join([repo, "contents", "accent.json"]) <> "?ref=#{ref}", + {:ok, config} when is_binary(config) <- fetch_content(path, token), + {:ok, %{"files" => files}} <- Jason.decode(config) do + files + else + _ -> [] + end + end + + defp fetch_trees(repo, token, ref) do + with path <- Path.join([repo, "git", "trees", ref]) <> "?recursive=1", + {:ok, %{body: %{"tree" => tree}}} <- file_server().get(path, headers(token)) do + filter_blob_file_only(tree) + else + _ -> [] + end + end + + defp filter_blob_file_only(files) do + Enum.filter(files, &(&1["type"] === "blob")) + end + + defp headers(token) do + [{"Authorization", "token #{token}"}] + end + + defp file_server, do: Application.get_env(:accent, :hook_github_file_server) +end diff --git a/lib/hook/consumers/github/add_translations.ex b/lib/hook/consumers/github/add_translations.ex new file mode 100644 index 000000000..379833491 --- /dev/null +++ b/lib/hook/consumers/github/add_translations.ex @@ -0,0 +1,71 @@ +defmodule Accent.Hook.Consumers.GitHub.AddTranslations do + alias Accent.Plugs.MovementContextParser + alias Movement.Builders.RevisionMerge, as: RevisionMergeBuilder + alias Movement.Context + alias Movement.Persisters.RevisionMerge, as: RevisionMergePersister + + alias Accent.Hook.Consumers.GitHub + + def persist(trees, configs, project, user, revision, payload, version) do + token = payload[:token] + + trees + |> group_by_matched_target_config(configs, revision) + |> Enum.reject(&(elem(&1, 0) === nil)) + |> Enum.flat_map(fn {config, files} -> + Enum.map(files, &build_context(&1, project, token, config["format"])) + end) + |> Enum.reject(&is_nil/1) + |> Enum.map(&assign_defaults(&1, user, revision, project, version)) + |> Enum.each(&persist_contexts/1) + end + + defp group_by_matched_target_config(files, configs, revision) do + configs + |> Enum.map(fn config -> + target = + config["target"] + |> Kernel.||("") + |> String.replace("%slug%", revision.language.slug) + |> String.replace("%original_file_name%", "*") + + Map.put(config, "matcher", ExMinimatch.compile(target)) + end) + |> GitHub.filter_by_patterns(files) + end + + defp persist_contexts(context) do + context + |> RevisionMergeBuilder.build() + |> RevisionMergePersister.persist() + end + + defp assign_defaults(context, user, revision, project, version) do + context + |> Context.assign(:user_id, user.id) + |> Context.assign(:revision, revision) + |> Context.assign(:project, project) + |> Context.assign(:version, version) + |> Context.assign(:merge_type, "smart") + |> Context.assign(:comparer, Movement.Comparer.comparer(:merge, "smart")) + end + + defp build_context(file, project, token, format) do + with {:ok, parser} <- Langue.parser_from_format(format), + document = %{id: id} when not is_nil(id) <- GitHub.movement_document(project, file["path"]), + document <- %{document | format: format}, + {:ok, file_content} <- GitHub.fetch_content(file["url"], token), + %{entries: entries} <- MovementContextParser.to_entries(document, file_content, parser) do + %Context{ + render: file_content, + entries: entries, + assigns: %{ + document: document, + document_update: %{} + } + } + else + _ -> nil + end + end +end diff --git a/lib/hook/consumers/github/file_server.ex b/lib/hook/consumers/github/file_server.ex new file mode 100644 index 000000000..ccfa7285f --- /dev/null +++ b/lib/hook/consumers/github/file_server.ex @@ -0,0 +1,3 @@ +defmodule Accent.Hook.Consumers.GitHub.FileServer do + @callback get(String.t(), list()) :: {:ok, String.t()} | {:error, any()} +end diff --git a/lib/hook/consumers/github/http.ex b/lib/hook/consumers/github/http.ex new file mode 100644 index 000000000..8f661d9ab --- /dev/null +++ b/lib/hook/consumers/github/http.ex @@ -0,0 +1,19 @@ +defmodule Accent.Hook.Consumers.GitHub.FileServer.HTTP do + use HTTPoison.Base + + @behaviour Accent.Hook.Consumers.GitHub.FileServer + + @base_url "https://api.github.com/repos/" + + def process_url(@base_url <> path), do: process_url(path) + def process_url(path), do: @base_url <> path + + def process_response_body(body) do + body + |> Jason.decode() + |> case do + {:ok, body} -> body + _ -> :error + end + end +end diff --git a/lib/hook/consumers/github/sync.ex b/lib/hook/consumers/github/sync.ex new file mode 100644 index 000000000..981c36ded --- /dev/null +++ b/lib/hook/consumers/github/sync.ex @@ -0,0 +1,64 @@ +defmodule Accent.Hook.Consumers.GitHub.Sync do + alias Accent.Plugs.MovementContextParser + alias Movement.Builders.ProjectSync, as: SyncBuilder + alias Movement.Context + alias Movement.Persisters.ProjectSync, as: SyncPersister + + alias Accent.Hook.Consumers.GitHub + + def persist(trees, configs, project, user, payload, version) do + token = payload[:token] + + trees + |> group_by_matched_source_config(configs) + |> Enum.reject(&(elem(&1, 0) === nil)) + |> Enum.flat_map(fn {config, files} -> + Enum.map(files, &build_context(&1, project, token, config["format"])) + end) + |> Enum.reject(&is_nil/1) + |> Enum.map(&assign_defaults(&1, user, project, version)) + |> Enum.each(&persist_contexts/1) + end + + defp group_by_matched_source_config(files, configs) do + configs + |> Enum.map(&Map.put(&1, "matcher", ExMinimatch.compile(&1["source"]))) + |> GitHub.filter_by_patterns(files) + end + + defp persist_contexts(context) do + context + |> SyncBuilder.build() + |> SyncPersister.persist() + end + + defp assign_defaults(context, user, project, version) do + context + |> Context.assign(:user_id, user.id) + |> Context.assign(:project, project) + |> Context.assign(:version, version) + |> Context.assign(:comparer, Movement.Comparer.comparer(:sync, "smart")) + end + + defp build_context(file, project, token, format) do + with {:ok, parser} <- Langue.parser_from_format(format), + document <- GitHub.movement_document(project, file["path"]), + document <- %{document | format: format}, + {:ok, file_content} <- GitHub.fetch_content(file["url"], token), + %{entries: entries, document: parsed_document} <- MovementContextParser.to_entries(document, file_content, parser) do + %Context{ + render: file_content, + entries: entries, + assigns: %{ + document: document, + document_update: %{ + top_of_the_file_comment: parsed_document.top_of_the_file_comment, + header: parsed_document.header + } + } + } + else + _ -> nil + end + end +end diff --git a/lib/hook/event_producer.ex b/lib/hook/event_producer.ex index b49ee7683..7003b10ef 100644 --- a/lib/hook/event_producer.ex +++ b/lib/hook/event_producer.ex @@ -11,21 +11,27 @@ defmodule Accent.Hook.EventProducer do {:producer, {:queue.new(), 0}, dispatcher: GenStage.BroadcastDispatcher} end - def handle_call({:notify, event}, from, {queue, demand}) do - dispatch_events(:queue.in({from, event}, queue), demand, []) + def handle_call({_, event}, from, {queue, pending_demand}) do + queue = :queue.in({from, event}, queue) + dispatch_events(queue, pending_demand, []) end - def handle_demand(incoming_demand, {queue, demand}) do - dispatch_events(queue, incoming_demand + demand, []) + def handle_demand(incoming_demand, {queue, pending_demand}) do + dispatch_events(queue, incoming_demand + pending_demand, []) + end + + defp dispatch_events(queue, 0, events) do + {:noreply, Enum.reverse(events), {queue, 0}} end defp dispatch_events(queue, demand, events) do - with d when d > 0 <- demand, - {{:value, {from, event}}, queue} <- :queue.out(queue) do - GenStage.reply(from, :ok) - dispatch_events(queue, demand - 1, [event | events]) - else - _ -> {:noreply, Enum.reverse(events), {queue, demand}} + case :queue.out(queue) do + {{:value, {from, event}}, queue} -> + GenStage.reply(from, :ok) + dispatch_events(queue, demand - 1, [event | events]) + + {:empty, queue} -> + {:noreply, Enum.reverse(events), {queue, demand}} end end end diff --git a/lib/hook/hook.ex b/lib/hook/hook.ex index c3f099c28..12ca511d2 100644 --- a/lib/hook/hook.ex +++ b/lib/hook/hook.ex @@ -1,6 +1,10 @@ defmodule Accent.Hook do - def fanout(context) do - broadcaster().fanout(context) + def notify(context) do + broadcaster().notify(context) + end + + def external_document_update(service, context) do + broadcaster().external_document_update(service, context) end defp broadcaster, do: Application.get_env(:accent, :hook_broadcaster) diff --git a/lib/hook/producers/github.ex b/lib/hook/producers/github.ex new file mode 100644 index 000000000..94d5a9753 --- /dev/null +++ b/lib/hook/producers/github.ex @@ -0,0 +1,3 @@ +defmodule Accent.Hook.Producers.GitHub do + use Accent.Hook.EventProducer +end diff --git a/lib/web/controllers/hook/github_controller.ex b/lib/web/controllers/hook/github_controller.ex new file mode 100644 index 000000000..a7c65c386 --- /dev/null +++ b/lib/web/controllers/hook/github_controller.ex @@ -0,0 +1,67 @@ +defmodule Accent.Hook.GitHubController do + use Plug.Builder + + import Canary.Plugs + import Ecto.Query, only: [first: 1] + + alias Accent.Hook.Context, as: HookContext + alias Accent.{Integration, Project, Repo} + alias Accent.Scopes.Integration, as: IntegrationScope + + plug(:filter_event_type) + plug(Plug.Assign, canary_action: :hook_update) + plug(:load_and_authorize_resource, model: Project, id_name: "project_id") + plug(:assign_payload) + plug(:update) + + def update(conn, _) do + Accent.Hook.external_document_update(:github, %HookContext{ + payload: conn.assigns[:payload], + project: conn.assigns[:project], + user: conn.assigns[:current_user] + }) + + send_resp(conn, :no_content, "") + end + + defp assign_payload(conn, _) do + with repository when is_binary(repository) <- conn.params["repository"]["full_name"], + ref when is_binary(ref) <- conn.params["ref"], + %{data: %{token: token, default_ref: default_ref}} <- repository_integration(conn.assigns[:project], repository) do + assign(conn, :payload, %{ + default_ref: default_ref, + ref: ref, + repository: repository, + token: token + }) + else + _ -> + conn + |> send_resp(:no_content, "") + |> halt() + end + end + + defp repository_integration(project, repository) do + Integration + |> IntegrationScope.from_project(project.id) + |> IntegrationScope.from_service("github") + |> IntegrationScope.from_data_repository(repository) + |> first() + |> Repo.one() + end + + defp filter_event_type(conn, _) do + conn + |> get_req_header("x-github-event") + |> case do + ["push"] -> + conn + + _ -> + conn + |> send_resp(:not_implemented, "") + |> halt() + end + end +end diff --git a/lib/web/controllers/merge_controller.ex b/lib/web/controllers/merge_controller.ex index 7d2752885..ad78ab667 100644 --- a/lib/web/controllers/merge_controller.ex +++ b/lib/web/controllers/merge_controller.ex @@ -61,7 +61,7 @@ defmodule Accent.MergeController do send_resp(conn, :ok, "") {:ok, _} -> - Accent.Hook.fanout(%HookContext{ + Accent.Hook.notify(%HookContext{ event: "merge", project: conn.assigns[:project], user: conn.assigns[:current_user], diff --git a/lib/web/controllers/peek_controller.ex b/lib/web/controllers/peek_controller.ex index baf810dfd..0af4b888a 100644 --- a/lib/web/controllers/peek_controller.ex +++ b/lib/web/controllers/peek_controller.ex @@ -57,7 +57,7 @@ defmodule Accent.PeekController do |> Map.get(:operations) |> Enum.group_by(&Map.get(&1, :revision_id)) - Accent.Hook.fanout(%HookContext{ + Accent.Hook.notify(%HookContext{ event: "peek_sync", project: conn.assigns[:project], user: conn.assigns[:current_user] @@ -101,7 +101,7 @@ defmodule Accent.PeekController do |> Map.get(:operations) |> Enum.group_by(&Map.get(&1, :revision_id)) - Accent.Hook.fanout(%HookContext{ + Accent.Hook.notify(%HookContext{ event: "peek_merge", project: conn.assigns[:project], user: conn.assigns[:current_user], diff --git a/lib/web/controllers/sync_controller.ex b/lib/web/controllers/sync_controller.ex index 8a0b5cbf0..9a5d5a4eb 100644 --- a/lib/web/controllers/sync_controller.ex +++ b/lib/web/controllers/sync_controller.ex @@ -50,7 +50,7 @@ defmodule Accent.SyncController do send_resp(conn, :ok, "") {:ok, {context, _operations}} -> - Accent.Hook.fanout(%HookContext{ + Accent.Hook.notify(%HookContext{ event: "sync", project: conn.assigns[:project], user: conn.assigns[:current_user], diff --git a/lib/web/plugs/assign_current_user.ex b/lib/web/plugs/assign_current_user.ex index 780769aab..f101ecc28 100644 --- a/lib/web/plugs/assign_current_user.ex +++ b/lib/web/plugs/assign_current_user.ex @@ -6,16 +6,29 @@ defmodule Accent.Plugs.AssignCurrentUser do def init(_), do: nil @doc """ - Takes a Plug.Conn and fetch the associated user giving the Authorization header. + Takes a Plug.Conn and fetch the associated user giving the Authorization HTTP header. + Fallbacks to the "authorization" query param to handle services without HTTP headers access (like webhooks). It assigns nil if any of the steps fails. """ def call(conn, _opts) do - user = + token = conn |> get_req_header("authorization") |> List.first() - |> UserAuthFetcher.fetch() + |> fallback_query_param_token(conn) + + user = UserAuthFetcher.fetch(token) assign(conn, :current_user, user) end + + defp fallback_query_param_token(token, _) when not is_nil(token), do: token + + defp fallback_query_param_token(nil, %{params: %{"authorization" => token}}) when is_binary(token) do + "Bearer " <> token + end + + defp fallback_query_param_token(_, _) do + nil + end end diff --git a/lib/web/router.ex b/lib/web/router.ex index 882135398..9ce0ac447 100644 --- a/lib/web/router.ex +++ b/lib/web/router.ex @@ -45,6 +45,8 @@ defmodule Accent.Router do # File export get("/export", ExportController, []) get("/jipt-export", ExportJIPTController, []) + + post("/hooks/github", Hook.GitHubController, [], as: :hooks_github) end scope "/", Accent do diff --git a/mix.exs b/mix.exs index 56df684a8..7ef06c37e 100644 --- a/mix.exs +++ b/mix.exs @@ -70,6 +70,7 @@ defmodule Accent.Mixfile do {:jason, "~> 1.0"}, {:erlsom, "~> 1.5"}, {:xml_builder, "~> 2.0"}, + {:ex_minimatch, "~> 0.0.1"}, # Errors {:sentry, "~> 7.0"}, diff --git a/mix.lock b/mix.lock index 3836bc7fc..b96926b94 100644 --- a/mix.lock +++ b/mix.lock @@ -26,7 +26,9 @@ "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "erlsom": {:hex, :erlsom, "1.5.0", "c5a5cdd0ee0e8dca62bcc4b13ff08da24fdefc16ccd8b25282a2fda2ba1be24a", [:rebar3], [], "hexpm"}, + "ex_brace_expansion": {:hex, :ex_brace_expansion, "0.0.2", "7574fd9497f3f045346dfd9517f10f237f4d39137bf42142b0fbdcd4bacbc6ed", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.12.0", "b774aabfede4af31c0301aece12371cbd25995a21bb3d71d66f5c2fe074c603f", [:mix], [{:earmark, "~> 0.2", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_minimatch": {:hex, :ex_minimatch, "0.0.1", "4b41726183c104ac227c5996f083ec370f97bd38c2232d74a847888c1bb715bc", [:mix], [{:ex_brace_expansion, "~> 0.0.1", [hex: :ex_brace_expansion, repo: "hexpm", optional: false]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.10.5", "7c912c4ec0715a6013647d835c87cde8154855b9b84e256bc7a63858d5f284e3", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, "fast_yaml": {:hex, :fast_yaml, "1.0.17", "e945ef64e0cb7c311c7b42804dbe32a24e13a2afc0ffe249b7e0f9f9ac08e176", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/rel/config/config.exs b/rel/config/config.exs index bfeb162dc..1d9369caf 100644 --- a/rel/config/config.exs +++ b/rel/config/config.exs @@ -30,6 +30,7 @@ config :accent, Accent.Repo, config :accent, force_ssl: Utilities.string_to_boolean(System.get_env("FORCE_SSL")), hook_broadcaster: Accent.Hook.Broadcaster, + hook_github_file_server: Accent.Hook.Consumers.GitHub.FileServer.HTTP, dummy_provider_enabled: true, restricted_domain: System.get_env("RESTRICTED_DOMAIN") diff --git a/test/graphql/resolvers/collaborator_test.exs b/test/graphql/resolvers/collaborator_test.exs index 02148d052..01774dc82 100644 --- a/test/graphql/resolvers/collaborator_test.exs +++ b/test/graphql/resolvers/collaborator_test.exs @@ -30,7 +30,7 @@ defmodule AccentTest.GraphQL.Resolvers.Collaborator do context = %{context: %{conn: %PlugConn{assigns: %{current_user: user}}}} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn _ -> :ok end) + |> expect(:notify, fn _ -> :ok end) {:ok, result} = Resolver.create(project, %{email: "test@example.com", role: "admin"}, context) diff --git a/test/graphql/resolvers/comment_test.exs b/test/graphql/resolvers/comment_test.exs index 8997c9c5d..8168fcdc6 100644 --- a/test/graphql/resolvers/comment_test.exs +++ b/test/graphql/resolvers/comment_test.exs @@ -37,7 +37,7 @@ defmodule AccentTest.GraphQL.Resolvers.Comment do context = %{context: %{conn: %PlugConn{assigns: %{current_user: user}}}} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn _ -> :ok end) + |> expect(:notify, fn _ -> :ok end) {:ok, result} = Resolver.create(translation, %{text: "First comment"}, context) diff --git a/test/hook/consumers/github_test.exs b/test/hook/consumers/github_test.exs new file mode 100644 index 000000000..feeee94dc --- /dev/null +++ b/test/hook/consumers/github_test.exs @@ -0,0 +1,288 @@ +defmodule AccentTest.Hook.Consumers.GitHub do + use Accent.RepoCase + + alias Accent.Hook.Consumers.GitHub, as: Consumer + alias Accent.Hook.Consumers.GitHub.FileServerMock + alias Accent.{Document, Integration, Language, Operation, ProjectCreator, Repo, Revision, Translation, User, Version} + + import Ecto.Query + + import Mox + setup :verify_on_exit! + + setup do + user = Repo.insert!(%User{email: "test@test.com"}) + language = Repo.insert!(%Language{name: "English", slug: Ecto.UUID.generate()}) + {:ok, project} = ProjectCreator.create(params: %{main_color: "#f00", name: "My project", language_id: language.id}, user: user) + document = Repo.insert!(%Document{project_id: project.id, path: "admin", format: "json"}) + + [project: project, document: document, user: user] + end + + def file do + Base.encode64(~S( + msgid "key" + msgstr "value" + )) + end + + test "sync default version on default_ref develop", %{project: project, user: user} do + config = + %{ + "files" => [ + %{ + "format" => "gettext", + "language" => "fr", + "source" => "priv/fr/**/*.po" + } + ] + } + |> Jason.encode!() + |> Base.encode64() + + FileServerMock + |> expect(:get, fn "accent/test-repo/contents/accent.json?ref=develop", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => config}}} + end) + |> expect(:get, fn "accent/test-repo/git/trees/develop?recursive=1", [{"Authorization", "token 1234"}] -> + {:ok, + %{ + body: %{ + "tree" => [ + %{"path" => "accent.json", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/1"}, + %{"path" => "Dockerfile", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/2"}, + %{"path" => "priv/fr", "type" => "tree", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/3"}, + %{"path" => "priv/fr", "type" => "tree", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/4"}, + %{"path" => "priv/fr/admin.po", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/5"}, + %{"path" => "priv/en/admin.po", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/6"} + ] + } + }} + end) + |> expect(:get, fn "https://api.github.com/repos/accent/test-repo/git/blobs/5", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => file()}}} + end) + + data = %{default_ref: "develop", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + event = %Accent.Hook.Context{ + project: project, + event: "push", + payload: %{ + default_ref: data.default_ref, + ref: "refs/heads/develop", + repository: data.repository, + token: data.token + } + } + + Consumer.handle_events([event], nil, []) + + batch_operation = + Operation + |> where([o], o.batch == true) + |> Repo.one() + + operation = + Operation + |> where([o], o.batch == false) + |> Repo.one() + + translation = + Translation + |> where([t], t.key == ^"key") + |> Repo.one() + + assert batch_operation.action === "sync" + assert operation.action === "new" + assert operation.translation_id === translation.id + assert translation.proposed_text === "value" + end + + test "dont sync when default ref does not match", %{project: project, user: user} do + data = %{default_ref: "master", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + event = %Accent.Hook.Context{ + project: project, + event: "push", + payload: %{ + default_ref: data.default_ref, + ref: "refs/heads/feature/my-feature", + repository: data.repository, + token: data.token + } + } + + Consumer.handle_events([event], nil, []) + + translation = + Translation + |> where([t], t.key == ^"key") + |> Repo.one() + + assert translation === nil + end + + test "sync tag version on matching ref tag", %{project: project, user: user} do + config = + %{ + "files" => [ + %{ + "format" => "gettext", + "language" => "fr", + "source" => "priv/fr/**/*.po" + } + ] + } + |> Jason.encode!() + |> Base.encode64() + + FileServerMock + |> expect(:get, fn "accent/test-repo/contents/accent.json?ref=v1.0.0", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => config}}} + end) + |> expect(:get, fn "accent/test-repo/git/trees/v1.0.0?recursive=1", [{"Authorization", "token 1234"}] -> + {:ok, + %{ + body: %{ + "tree" => [ + %{"path" => "accent.json", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/1"}, + %{"path" => "Dockerfile", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/2"}, + %{"path" => "priv/fr", "type" => "tree", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/3"}, + %{"path" => "priv/fr", "type" => "tree", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/4"}, + %{"path" => "priv/fr/admin.po", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/5"}, + %{"path" => "priv/en/admin.po", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/6"} + ] + } + }} + end) + |> expect(:get, fn "https://api.github.com/repos/accent/test-repo/git/blobs/5", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => file()}}} + end) + + version = Repo.insert!(%Version{project_id: project.id, user_id: user.id, tag: "v1.0.0", name: "First release"}) + data = %{default_ref: "master", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + event = %Accent.Hook.Context{ + project: project, + event: "push", + payload: %{ + default_ref: data.default_ref, + ref: "refs/tags/v1.0.0", + repository: data.repository, + token: data.token + } + } + + Consumer.handle_events([event], nil, []) + + batch_operation = + Operation + |> where([o], o.batch == true and o.version_id == ^version.id) + |> Repo.one() + + operation = + Operation + |> where([o], o.batch == false and o.version_id == ^version.id) + |> Repo.one() + + translation = + Translation + |> where([t], t.key == ^"key" and t.version_id == ^version.id) + |> Repo.one() + + assert batch_operation.action === "sync" + assert operation.action === "new" + assert operation.translation_id === translation.id + assert translation.proposed_text === "value" + end + + test "add translations default version on default_ref develop", %{project: project, document: document, user: user} do + language_slug = Ecto.UUID.generate() + language = Repo.insert!(%Language{name: "Other french", slug: language_slug}) + revision = Repo.insert!(%Revision{project_id: project.id, master: false, language: language}) + translation = Repo.insert!(%Translation{revision_id: revision.id, document_id: document.id, key: "key", proposed_text: "a", corrected_text: "a"}) + + config = + %{ + "files" => [ + %{ + "format" => "gettext", + "language" => "fr", + "source" => "priv/fr/**/*.po", + "target" => "priv/%slug%/**/%original_file_name%.po" + } + ] + } + |> Jason.encode!() + |> Base.encode64() + + FileServerMock + |> expect(:get, fn "accent/test-repo/contents/accent.json?ref=develop", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => config}}} + end) + |> expect(:get, fn "accent/test-repo/git/trees/develop?recursive=1", [{"Authorization", "token 1234"}] -> + {:ok, + %{ + body: %{ + "tree" => [ + %{"path" => "accent.json", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/1"}, + %{"path" => "Dockerfile", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/2"}, + %{"path" => "priv/#{language_slug}", "type" => "tree", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/4"}, + %{"path" => "priv/#{language_slug}/admin.po", "type" => "blob", "url" => "https://api.github.com/repos/accent/test-repo/git/blobs/6"} + ] + } + }} + end) + |> expect(:get, fn "https://api.github.com/repos/accent/test-repo/git/blobs/6", [{"Authorization", "token 1234"}] -> + {:ok, %{body: %{"content" => file()}}} + end) + + data = %{default_ref: "develop", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + event = %Accent.Hook.Context{ + project: project, + event: "push", + payload: %{ + default_ref: data.default_ref, + ref: "refs/heads/develop", + repository: data.repository, + token: data.token + } + } + + Consumer.handle_events([event], nil, []) + + batch_operation = + Operation + |> where([o], o.batch == true) + |> Repo.one() + + operation = + Operation + |> where([o], o.batch == false) + |> Repo.one() + + updated_translation = + Translation + |> where([t], t.key == ^"key") + |> Repo.one() + + assert batch_operation.action === "merge" + assert operation.action === "merge_on_proposed" + assert operation.translation_id === translation.id + + assert operation.previous_translation === %Accent.PreviousTranslation{ + corrected_text: "a", + proposed_text: "a", + value_type: "string" + } + + assert updated_translation.conflicted_text === "a" + assert updated_translation.proposed_text === "value" + end +end diff --git a/test/scopes/integration_test.exs b/test/scopes/integration_test.exs new file mode 100644 index 000000000..4f455afce --- /dev/null +++ b/test/scopes/integration_test.exs @@ -0,0 +1,4 @@ +defmodule AccentTest.Scopes.Integration do + use ExUnit.Case, async: true + doctest Accent.Scopes.Integration +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 256b9ffa5..1b6af4552 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -1 +1,2 @@ Mox.defmock(Accent.Hook.BroadcasterMock, for: Accent.Hook.Broadcaster) +Mox.defmock(Accent.Hook.Consumers.GitHub.FileServerMock, for: Accent.Hook.Consumers.GitHub.FileServer) diff --git a/test/web/controllers/hook/github_controller_test.exs b/test/web/controllers/hook/github_controller_test.exs new file mode 100644 index 000000000..e7298dab0 --- /dev/null +++ b/test/web/controllers/hook/github_controller_test.exs @@ -0,0 +1,111 @@ +defmodule AccentTest.Hook.GitHubController do + use Accent.ConnCase + + import Mox + setup :verify_on_exit! + + alias Accent.Hook.Context, as: HookContext + + alias Accent.{ + AccessToken, + Collaborator, + Integration, + Project, + Repo, + User + } + + @user %User{email: "test@test.com"} + + setup do + user = Repo.insert!(@user) + access_token = %AccessToken{user_id: user.id, token: "test-token"} |> Repo.insert!() + project = %Project{main_color: "#f00", name: "My project"} |> Repo.insert!() + %Collaborator{project_id: project.id, user_id: user.id, role: "bot"} |> Repo.insert!() + + {:ok, [access_token: access_token, user: user, project: project]} + end + + test "broadcast event on push", %{user: user, access_token: access_token, conn: conn, project: project} do + params = %{ + "ref" => "refs/heads/master", + "repository" => %{ + "full_name" => "accent/test-repo" + } + } + + data = %{default_ref: "master", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + payload = %{ + default_ref: "master", + ref: "refs/heads/master", + repository: "accent/test-repo", + token: "1234" + } + + Accent.Hook.BroadcasterMock + |> expect(:external_document_update, fn :github, %HookContext{payload: ^payload} -> :ok end) + + response = + conn + |> put_req_header("x-github-event", "push") + |> post(hooks_github_path(conn, []) <> "?authorization=#{access_token.token}&project_id=#{project.id}", params) + + assert response.status == 204 + end + + test "don’t broadcast event on other event", %{user: user, access_token: access_token, conn: conn, project: project} do + params = %{ + "ref" => "refs/heads/master", + "repository" => %{ + "full_name" => "accent/test-repo" + } + } + + data = %{default_ref: "master", repository: "accent/test-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + response = + conn + |> put_req_header("x-github-event", "pull_request_comment") + |> post(hooks_github_path(conn, []) <> "?authorization=#{access_token.token}&project_id=#{project.id}", params) + + assert response.status == 501 + end + + test "don’t broadcast event on non existing integration", %{access_token: access_token, conn: conn, project: project} do + params = %{ + "ref" => "refs/heads/master", + "repository" => %{ + "full_name" => "accent/test-repo" + } + } + + response = + conn + |> put_req_header("x-github-event", "push") + |> post(hooks_github_path(conn, []) <> "?authorization=#{access_token.token}&project_id=#{project.id}", params) + + assert response.status == 204 + end + + test "don’t broadcast event on non matching integration", %{user: user, access_token: access_token, conn: conn, project: project} do + params = %{ + "ref" => "refs/heads/master", + "repository" => %{ + "full_name" => "accent/test-repo" + } + } + + data = %{default_ref: "master", repository: "accent/other-repo", token: "1234"} + Repo.insert!(%Integration{project_id: project.id, user_id: user.id, service: "github", data: data}) + + response = + conn + |> put_req_header("x-github-event", "push") + |> post(hooks_github_path(conn, []) <> "?authorization=#{access_token.token}&project_id=#{project.id}", params) + + assert response.status == 204 + end +end diff --git a/test/web/controllers/merge_controller_test.exs b/test/web/controllers/merge_controller_test.exs index 578aaa31b..37962ec8c 100644 --- a/test/web/controllers/merge_controller_test.exs +++ b/test/web/controllers/merge_controller_test.exs @@ -43,7 +43,7 @@ defmodule AccentTest.MergeController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "merge"} -> :ok end) + |> expect(:notify, fn %{event: "merge"} -> :ok end) response = conn @@ -87,7 +87,7 @@ defmodule AccentTest.MergeController do body = %{file: file(), merge_type: "force", project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "merge"} -> :ok end) + |> expect(:notify, fn %{event: "merge"} -> :ok end) response = conn @@ -112,7 +112,7 @@ defmodule AccentTest.MergeController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "merge"} -> :ok end) + |> expect(:notify, fn %{event: "merge"} -> :ok end) response = conn diff --git a/test/web/controllers/peek_controller_test.exs b/test/web/controllers/peek_controller_test.exs index d9ccfb31b..f3150e999 100644 --- a/test/web/controllers/peek_controller_test.exs +++ b/test/web/controllers/peek_controller_test.exs @@ -41,7 +41,7 @@ defmodule AccentTest.PeekController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "peek_merge"} -> :ok end) + |> expect(:notify, fn %{event: "peek_merge"} -> :ok end) response = conn @@ -68,7 +68,7 @@ defmodule AccentTest.PeekController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "peek_merge"} -> :ok end) + |> expect(:notify, fn %{event: "peek_merge"} -> :ok end) response = conn @@ -95,7 +95,7 @@ defmodule AccentTest.PeekController do body = %{file: file(), merge_type: "passive", project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "peek_merge"} -> :ok end) + |> expect(:notify, fn %{event: "peek_merge"} -> :ok end) response = conn @@ -114,7 +114,7 @@ defmodule AccentTest.PeekController do body = %{file: file(), merge_type: "force", project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "peek_merge"} -> :ok end) + |> expect(:notify, fn %{event: "peek_merge"} -> :ok end) response = conn @@ -141,7 +141,7 @@ defmodule AccentTest.PeekController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: document.format, document_path: document.path} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn %{event: "peek_sync"} -> :ok end) + |> expect(:notify, fn %{event: "peek_sync"} -> :ok end) response = conn diff --git a/test/web/controllers/sync_controller_test.exs b/test/web/controllers/sync_controller_test.exs index f80d35086..2830e4836 100644 --- a/test/web/controllers/sync_controller_test.exs +++ b/test/web/controllers/sync_controller_test.exs @@ -39,7 +39,7 @@ defmodule AccentTest.SyncController do body = %{file: file(), project_id: project.id, language: language.slug, document_format: "json", document_path: "simple"} Accent.Hook.BroadcasterMock - |> expect(:fanout, fn _ -> :ok end) + |> expect(:notify, fn _ -> :ok end) response = conn