Skip to content

Commit

Permalink
add change detection to stack run creation process
Browse files Browse the repository at this point in the history
  • Loading branch information
michaeljguarino committed May 5, 2024
1 parent 96d46dc commit 9512502
Show file tree
Hide file tree
Showing 9 changed files with 94 additions and 18 deletions.
2 changes: 2 additions & 0 deletions lib/console.ex
Original file line number Diff line number Diff line change
Expand Up @@ -218,5 +218,7 @@ defmodule Console do
end
end

def lines(str), do: String.split(str, ~r/\R/)

def storage, do: Console.Storage.Git
end
6 changes: 6 additions & 0 deletions lib/console/deployments/git/agent.ex
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ defmodule Console.Deployments.Git.Agent do

def sha(pid, ref), do: GenServer.call(pid, {:sha, ref}, 30_000)

def changes(pid, sha1, sha2, folder), do: GenServer.call(pid, {:changes, sha1, sha2, folder}, 30_000)

def kick(pid), do: send(pid, :pull)

def start(%GitRepository{} = repo) do
Expand Down Expand Up @@ -83,6 +85,10 @@ defmodule Console.Deployments.Git.Agent do
{:reply, Cache.commit(cache, ref), state}
end

def handle_call({:changes, sha1, sha2, folder}, _, %State{cache: cache} = state) do
{:reply, Cache.changes(cache, sha1, sha2, folder), state}
end

def handle_call({:fetch, %Service.Git{} = ref}, _, %State{cache: cache} = state) do
case Cache.fetch(cache, ref) do
{:ok, %Cache.Line{file: f}, cache} -> {:reply, File.open(f), %{state | cache: cache}}
Expand Down
2 changes: 2 additions & 0 deletions lib/console/deployments/git/cache.ex
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ defmodule Console.Deployments.Git.Cache do
end
end

def changes(%__MODULE__{git: g}, sha1, sha2, folder), do: file_changes(g, sha1, sha2, folder)

defp new_line(cache, repo, sha, path, filter) do
with {:ok, _} <- git(repo, "checkout", [sha]),
{:ok, msg} <- msg(repo),
Expand Down
7 changes: 7 additions & 0 deletions lib/console/deployments/git/cmd.ex
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ defmodule Console.Deployments.Git.Cmd do
do: {:ok, String.trim(sha)}
end

def file_changes(repo, sha1, sha2, folder) do
case git(repo, "--no-pager", ["diff", "--name-only", "#{sha1}", "#{sha2}", "--", folder]) do
{:ok, res} -> {:ok, String.trim(res) |> Console.lines()}
_ -> {:ok, :pass}
end
end

def branches(%GitRepository{} = repo) do
with {:ok, res} <- git(repo, "branch", ["-r"]) do
split_and_trim(res)
Expand Down
5 changes: 5 additions & 0 deletions lib/console/deployments/git/discovery.ex
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ defmodule Console.Deployments.Git.Discovery do
do: Agent.sha(pid, ref)
end

def changes(%GitRepository{} = repo, sha1, sha2, folder) do
with {:ok, pid} <- find(repo),
do: Agent.changes(pid, sha1, sha2, folder)
end

@spec docs(Service.t) :: {:ok, File.t} | error
def docs(%Service{} = svc) do
%{repository: repo} = Console.Repo.preload(svc, [:repository])
Expand Down
41 changes: 28 additions & 13 deletions lib/console/deployments/stacks.ex
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,13 @@ defmodule Console.Deployments.Stacks do
def poll(%Stack{delete_run_id: id}) when is_binary(id),
do: {:error, "stack is deleting"}

def poll(%Stack{sha: sha} = stack) do
def poll(%Stack{sha: sha, git: git} = stack) do
%{repository: repo} = stack = Repo.preload(stack, @poll_preloads)
case Discovery.sha(repo, stack.git.ref) do
case Discovery.sha(repo, git.ref) do
{:ok, ^sha} -> {:error, "no new commit in repo"}
{:ok, new_sha} -> create_run(stack, new_sha)
{:ok, new_sha} ->
with {:ok, new_sha} <- new_changes(repo, git, sha, new_sha),
do: create_run(stack, new_sha)
err -> err
end
end
Expand All @@ -205,21 +207,31 @@ defmodule Console.Deployments.Stacks do
case Discovery.sha(repo, ref) do
{:ok, ^sha} -> {:error, "no new commit in repo for branch #{ref}"}
{:ok, new_sha} ->
start_transaction()
|> add_operation(:run, fn _ ->
create_run(stack, new_sha, %{pull_request_id: pr.id, dry_run: true})
end)
|> add_operation(:pr, fn _ ->
Ecto.Changeset.change(pr, %{ref: new_sha})
|> Repo.update()
end)
|> execute(extract: :run)
with {:ok, new_sha} <- new_changes(repo, stack.git, sha, new_sha) do
start_transaction()
|> add_operation(:run, fn _ ->
create_run(stack, new_sha, %{pull_request_id: pr.id, dry_run: true})
end)
|> add_operation(:pr, fn _ ->
Ecto.Changeset.change(pr, %{ref: new_sha})
|> Repo.update()
end)
|> execute(extract: :run)
end
err -> err
end
end

def poll(_), do: {:error, "invalid parent"}

defp new_changes(repo, %{folder: folder}, sha1, sha2) do
case Discovery.changes(repo, sha1, sha2, folder) do
{:ok, [_ | _]} -> {:ok, sha2}
{:ok, :pass} -> {:ok, sha2}
_ -> {:error, "no changes within #{folder}"}
end
end

@doc """
Creates a new run for the stack with the given sha and optional additional attrs
"""
Expand All @@ -241,14 +253,17 @@ defmodule Console.Deployments.Stacks do
|> Repo.insert()
end)
|> add_operation(:stack, fn %{run: run} ->
Ecto.Changeset.change(stack, %{sha: sha})
Ecto.Changeset.change(stack, sha_attrs(run, sha))
|> Stack.delete_changeset(delete_run(stack, run))
|> Repo.update()
end)
|> execute(extract: :run)
|> notify(:create)
end

defp sha_attrs(%StackRun{dry_run: true}, _sha), do: %{}
defp sha_attrs(%StackRun{}, sha), do: %{sha: sha}

defp delete_run(%Stack{deleted_at: d}, %{id: id}) when not is_nil(d),
do: %{delete_run_id: id}
defp delete_run(_, _), do: %{}
Expand Down
6 changes: 5 additions & 1 deletion test/console/deployments/cron_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,12 @@ defmodule Console.Deployments.CronTest do

describe "#poll_stacks/0" do
test "it can generate new stack runs" do
stack = insert(:stack, environment: [%{name: "ENV", value: "1"}], files: [%{path: "test.txt", content: "test"}])
stack = insert(:stack,
environment: [%{name: "ENV", value: "1"}], files: [%{path: "test.txt", content: "test"}],
git: %{ref: "main", folder: "terraform"}
)
expect(Console.Deployments.Git.Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end)
expect(Console.Deployments.Git.Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"]} end)

Cron.poll_stacks()

Expand Down
6 changes: 4 additions & 2 deletions test/console/deployments/pubsub/recurse_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -361,8 +361,9 @@ defmodule Console.Deployments.PubSub.RecurseTest do

describe "StackCreated" do
test "it will poll the stack" do
stack = insert(:stack)
stack = insert(:stack, git: %{folder: "terraform", ref: "main"})
expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end)
expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"]} end)

event = %PubSub.StackCreated{item: stack}
{:ok, run} = Recurse.handle_event(event)
Expand All @@ -377,8 +378,9 @@ defmodule Console.Deployments.PubSub.RecurseTest do

describe "StackUpdated" do
test "it will poll the stack" do
stack = insert(:stack)
stack = insert(:stack, git: %{folder: "terraform", ref: "main"})
expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end)
expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"]} end)

event = %PubSub.StackUpdated{item: stack}
{:ok, run} = Recurse.handle_event(event)
Expand Down
37 changes: 35 additions & 2 deletions test/console/deployments/stacks_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,13 @@ defmodule Console.Deployments.StacksTest do

describe "#poll/1" do
test "it can create a new run when the sha changes" do
stack = insert(:stack, environment: [%{name: "ENV", value: "1"}], files: [%{path: "test.txt", content: "test"}])
stack = insert(:stack,
environment: [%{name: "ENV", value: "1"}],
files: [%{path: "test.txt", content: "test"}],
git: %{ref: "main", folder: "terraform"}
)
expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end)
expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"]} end)

{:ok, run} = Stacks.poll(stack)

Expand Down Expand Up @@ -175,9 +180,14 @@ defmodule Console.Deployments.StacksTest do
end

test "it can create a new run from a pr the sha changes" do
stack = insert(:stack, environment: [%{name: "ENV", value: "1"}], files: [%{path: "test.txt", content: "test"}])
stack = insert(:stack,
environment: [%{name: "ENV", value: "1"}],
files: [%{path: "test.txt", content: "test"}],
git: %{ref: "main", folder: "terraform"}
)
pr = insert(:pull_request, stack: stack)
expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end)
expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"]} end)

{:ok, run} = Stacks.poll(pr)

Expand Down Expand Up @@ -431,3 +441,26 @@ defmodule Console.Deployments.StacksTest do
end
end
end

defmodule Console.Deployments.StacksSyncTest do
use Console.DataCase, async: false
alias Console.Deployments.Stacks

describe "#poll/1" do
test "it will create runs when it detects changes" do
git = insert(:git_repository, url: "https://github.com/pluralsh/console.git")
stack = insert(:stack,
repository: git,
git: %{ref: "master", folder: "charts"},
sha: "e136726eb7f3ef1d3578b8b250b1fc1957331a84"
)

{:ok, run} = Stacks.poll(stack)

assert run.status == :queued
refute run.dry_run
assert run.stack_id == stack.id
refute run.git.ref == stack.git.ref
end
end
end

0 comments on commit 9512502

Please sign in to comment.