Merge bc67c79 into 51ceec7

config/test.exs

@@ -23,3 +23,4 @@ config :accent, Accent.Mailer,
   adapter: Bamboo.TestAdapter
 
 config :accent, hook_broadcaster: Accent.Hook.BroadcasterMock
+config :accent, hook_github_file_server: Accent.Hook.Consumers.GitHub.FileServerMock

lib/accent.ex

@@ -11,6 +11,8 @@ defmodule Accent do
       supervisor(Accent.Endpoint, []),
       # Start the Ecto repository
       worker(Accent.Repo, []),
+      worker(Accent.Hook.Producers.GitHub, []),
+      worker(Accent.Hook.Consumers.GitHub, []),
       worker(Accent.Hook.Producers.Email, []),
       worker(Accent.Hook.Consumers.Email, mailer: Accent.Mailer),
       worker(Accent.Hook.Producers.Websocket, []),

lib/accent/auth/role_abilities.ex

@@ -41,6 +41,7 @@ defmodule Accent.RoleAbilities do
     peek_merge
     merge
     sync
+    hook_update
   )a ++ @any_actions
 
   @developer_actions ~w(

lib/accent/schemas/integration.ex

@@ -7,6 +7,9 @@ defmodule Accent.Integration do
 
     embeds_one(:data, IntegrationData, on_replace: :update) do
       field(:url)
+      field(:repository)
+      field(:token)
+      field(:default_ref, :string, default: "master")
     end
 
     belongs_to(:project, Accent.Project)

lib/accent/schemas/project.ex

@@ -9,6 +9,7 @@ defmodule Accent.Project do
 
     has_many(:integrations, Accent.Integration)
     has_many(:revisions, Accent.Revision)
+    has_many(:target_revisions, Accent.Revision, where: [master: false])
     has_many(:versions, Accent.Version)
     has_many(:operations, Accent.Operation)
     has_many(:collaborators, Accent.Collaborator)

lib/accent/scopes/integration.ex

@@ -0,0 +1,36 @@
+defmodule Accent.Scopes.Integration do
+  import Ecto.Query, only: [from: 2]
+
+  @doc """
+  ## Examples
+
+    iex> Accent.Scopes.Integration.from_project(Accent.Integration, "test")
+    #Ecto.Query<from i0 in Accent.Integration, where: i0.project_id == ^\"test\">
+  """
+  @spec from_project(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t()
+  def from_project(query, project_id) do
+    from(query, where: [project_id: ^project_id])
+  end
+
+  @doc """
+  ## Examples
+
+    iex> Accent.Scopes.Integration.from_service(Accent.Integration, "test")
+    #Ecto.Query<from i0 in Accent.Integration, where: i0.service == ^\"test\">
+  """
+  @spec from_service(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t()
+  def from_service(query, service) do
+    from(query, where: [service: ^service])
+  end
+
+  @doc """
+  ## Examples
+
+    iex> Accent.Scopes.Integration.from_data_repository(Accent.Integration, "test")
+    #Ecto.Query<from i0 in Accent.Integration, where: fragment(\"?->>'repository' = ?\", i0.data, ^\"test\")>
+  """
+  @spec from_data_repository(Ecto.Queryable.t(), String.t()) :: Ecto.Queryable.t()
+  def from_data_repository(query, repository) do
+    from(i in query, where: fragment("?->>'repository' = ?", i.data, ^repository))
+  end
+end

lib/graphql/resolvers/collaborator.ex

@@ -22,7 +22,7 @@ defmodule Accent.GraphQL.Resolvers.Collaborator do
 
     case CollaboratorCreator.create(params) do
       {:ok, collaborator} ->
-        Accent.Hook.fanout(%Hook.Context{
+        Accent.Hook.notify(%Hook.Context{
           event: "create_collaborator",
           project: project,
           user: info.context[:conn].assigns[:current_user],

lib/graphql/resolvers/comment.ex

@@ -27,7 +27,7 @@ defmodule Accent.GraphQL.Resolvers.Comment do
       {:ok, comment} ->
         comment = Repo.preload(comment, [:user, translation: [revision: :project]])
 
-        Accent.Hook.fanout(%Hook.Context{
+        Accent.Hook.notify(%Hook.Context{
           event: "create_comment",
           project: comment.translation.revision.project,
           user: info.context[:conn].assigns[:current_user],

lib/hook/broadcaster.ex

@@ -1,16 +1,22 @@
 defmodule Accent.Hook.Broadcaster do
-  @producers [
+  @notifiers [
     Accent.Hook.Producers.Email,
     Accent.Hook.Producers.Websocket,
     Accent.Hook.Producers.Slack
   ]
 
-  @callback fanout(Accent.Hook.Context.t()) :: no_return()
+  @callback notify(Accent.Hook.Context.t()) :: no_return()
+  @callback external_document_update(:github, Accent.Hook.Context.t()) :: no_return()
+
   @timeout 10_000
 
-  def fanout(context = %Accent.Hook.Context{}) do
-    for producer <- @producers do
+  def notify(context = %Accent.Hook.Context{}) do
+    for producer <- @notifiers do
       GenStage.call(producer, {:notify, context}, @timeout)
     end
   end
+
+  def external_document_update(:github, context = %Accent.Hook.Context{}) do
+    GenStage.call(Accent.Hook.Producers.GitHub, {:external_document_update, context}, @timeout)
+  end
 end

lib/hook/consumers/github.ex

@@ -0,0 +1,140 @@
+defmodule Accent.Hook.Consumers.GitHub do
+  @moduledoc """
+  From a project’s integration association and a webhook event from GitHub,
+  sync and add translation as if the operations were made manually by a user.
+
+  """
+  use Accent.Hook.EventConsumer, subscribe_to: [Accent.Hook.Producers.GitHub]
+
+  alias Accent.{Document, Repo, Version}
+  alias Accent.Hook.Consumers.GitHub.AddTranslations
+  alias Accent.Hook.Consumers.GitHub.Sync
+  alias Accent.Hook.Context
+  alias Accent.Plugs.MovementContextParser
+  alias Accent.Scopes.Document, as: DocumentScope
+  alias Accent.Scopes.Version, as: VersionScope
+
+  def handle_events(events, _from, state) do
+    Enum.each(events, &handle_event/1)
+
+    {:noreply, [], state}
+  end
+
+  defp handle_event(%Context{user: user, project: project, payload: payload}) do
+    payload[:ref]
+    |> ref_to_version(payload[:default_ref], project)
+    |> sync_and_add_translations(project, user, payload)
+  end
+
+  defp sync_and_add_translations({:ok, version}, project, user, payload) do
+    repo = payload[:repository]
+    token = payload[:token]
+
+    ref = (version && version.tag) || payload[:default_ref]
+    configs = fetch_config(repo, token, ref)
+    trees = fetch_trees(repo, token, ref)
+
+    revisions =
+      project
+      |> Ecto.assoc(:target_revisions)
+      |> Repo.all()
+      |> Repo.preload(:language)
+
+    Sync.persist(trees, configs, project, user, payload, version)
+
+    Enum.each(revisions, fn revision ->
+      AddTranslations.persist(trees, configs, project, user, revision, payload, version)
+    end)
+  end
+
+  defp sync_and_add_translations(_, _, _, _), do: :ok
+
+  def filter_by_patterns(patterns, files) do
+    Enum.group_by(files, fn file -> Enum.find(patterns, &ExMinimatch.match(&1["matcher"], file["path"])) end)
+  end
+
+  def movement_document(project, path) do
+    path =
+      path
+      |> Path.basename()
+      |> MovementContextParser.extract_path_from_filename()
+
+    Document
+    |> DocumentScope.from_path(path)
+    |> DocumentScope.from_project(project.id)
+    |> Repo.one()
+    |> Kernel.||(%Document{project_id: project.id, path: path})
+  end
+
+  def fetch_content(path, token) do
+    with {:ok, %{body: %{"content" => content}}} <- file_server().get(path, headers(token)),
+         decoded_contents <-
+           content
+           |> String.split("\n")
+           |> Enum.reject(&(&1 === ""))
+           |> Enum.map(&Base.decode64/1),
+         true <- Enum.all?(decoded_contents, &match?({:ok, _}, &1)),
+         decoded_content <-
+           decoded_contents
+           |> Enum.map(&elem(&1, 1))
+           |> Enum.join("") do
+      {:ok, decoded_content}
+    else
+      _ -> {:ok, nil}
+    end
+  end
+
+  def ref_to_version(ref, default_ref, project) do
+    default_version(ref, default_ref) || version_from_ref(ref, project)
+  end
+
+  defp default_version(ref, default_ref) do
+    case Regex.named_captures(~r/refs\/heads\/(?<branch>.+)/, ref) do
+      %{"branch" => ^default_ref} -> {:ok, nil}
+      _ -> nil
+    end
+  end
+
+  defp version_from_ref(ref, project) do
+    with %{"tag" => tag} <- Regex.named_captures(~r/refs\/tags\/(?<tag>.+)/, ref),
+         version = %Version{} <-
+           Version
+           |> VersionScope.from_project(project.id)
+           |> VersionScope.from_tag(tag)
+           |> Repo.one() do
+      {:ok, version}
+    else
+      _ ->
+        nil
+    end
+  end
+
+  defp fetch_config(repo, token, ref) do
+    with path <- Path.join([repo, "contents", "accent.json"]) <> "?ref=#{ref}",
+         {:ok, config} when is_binary(config) <- fetch_content(path, token),
+         {:ok, %{"files" => files}} <- Jason.decode(config) do
+      files
+    else
+      _ -> []
+    end
+  end
+
+  defp fetch_trees(repo, token, ref) do
+    with path <- Path.join([repo, "git", "trees", ref]) <> "?recursive=1",
+         {:ok, %{body: %{"tree" => tree}}} <- file_server().get(path, headers(token)) do
+      filter_blob_file_only(tree)
+    else
+      _ -> []
+    end
+  end
+
+  defp filter_blob_file_only(files) do
+    Enum.filter(files, &(&1["type"] === "blob"))
+  end
+
+  defp headers(token) do
+    [{"Authorization", "token #{token}"}]
+  end
+
+  defp file_server, do: Application.get_env(:accent, :hook_github_file_server)
+end

lib/hook/consumers/github/add_translations.ex

@@ -0,0 +1,71 @@
+defmodule Accent.Hook.Consumers.GitHub.AddTranslations do
+  alias Accent.Plugs.MovementContextParser
+  alias Movement.Builders.RevisionMerge, as: RevisionMergeBuilder
+  alias Movement.Context
+  alias Movement.Persisters.RevisionMerge, as: RevisionMergePersister
+
+  alias Accent.Hook.Consumers.GitHub
+
+  def persist(trees, configs, project, user, revision, payload, version) do
+    token = payload[:token]
+
+    trees
+    |> group_by_matched_target_config(configs, revision)
+    |> Enum.reject(&(elem(&1, 0) === nil))
+    |> Enum.flat_map(fn {config, files} ->
+      Enum.map(files, &build_context(&1, project, token, config["format"]))
+    end)
+    |> Enum.reject(&is_nil/1)
+    |> Enum.map(&assign_defaults(&1, user, revision, project, version))
+    |> Enum.each(&persist_contexts/1)
+  end
+
+  defp group_by_matched_target_config(files, configs, revision) do
+    configs
+    |> Enum.map(fn config ->
+      target =
+        config["target"]
+        |> Kernel.||("")
+        |> String.replace("%slug%", revision.language.slug)
+        |> String.replace("%original_file_name%", "*")
+
+      Map.put(config, "matcher", ExMinimatch.compile(target))
+    end)
+    |> GitHub.filter_by_patterns(files)
+  end
+
+  defp persist_contexts(context) do
+    context
+    |> RevisionMergeBuilder.build()
+    |> RevisionMergePersister.persist()
+  end
+
+  defp assign_defaults(context, user, revision, project, version) do
+    context
+    |> Context.assign(:user_id, user.id)
+    |> Context.assign(:revision, revision)
+    |> Context.assign(:project, project)
+    |> Context.assign(:version, version)
+    |> Context.assign(:merge_type, "smart")
+    |> Context.assign(:comparer, Movement.Comparer.comparer(:merge, "smart"))
+  end
+
+  defp build_context(file, project, token, format) do
+    with {:ok, parser} <- Langue.parser_from_format(format),
+         document = %{id: id} when not is_nil(id) <- GitHub.movement_document(project, file["path"]),
+         document <- %{document | format: format},
+         {:ok, file_content} <- GitHub.fetch_content(file["url"], token),
+         %{entries: entries} <- MovementContextParser.to_entries(document, file_content, parser) do
+      %Context{
+        render: file_content,
+        entries: entries,
+        assigns: %{
+          document: document,
+          document_update: %{}
+        }
+      }
+    else
+      _ -> nil
+    end
+  end
+end

lib/hook/consumers/github/file_server.ex

@@ -0,0 +1,3 @@
+defmodule Accent.Hook.Consumers.GitHub.FileServer do
+  @callback get(String.t(), list()) :: {:ok, String.t()} | {:error, any()}
+end

lib/hook/consumers/github/http.ex

@@ -0,0 +1,19 @@
+defmodule Accent.Hook.Consumers.GitHub.FileServer.HTTP do
+  use HTTPoison.Base
+
+  @behaviour Accent.Hook.Consumers.GitHub.FileServer
+
+  @base_url "https://api.github.com/repos/"
+
+  def process_url(@base_url <> path), do: process_url(path)
+  def process_url(path), do: @base_url <> path
+
+  def process_response_body(body) do
+    body
+    |> Jason.decode()
+    |> case do
+      {:ok, body} -> body
+      _ -> :error
+    end
+  end
+end