Implement terraform http state backend

This along with a agent-specified `_override.tf` file will allow us to serve as a built-in state store for terraform.

assets/src/generated/graphql.ts

@@ -1996,6 +1996,8 @@ export type InfrastructureStack = {
   /** A type for the stack, specifies the tool to use to apply it */
   type: StackType;
   updatedAt?: Maybe<Scalars['DateTime']['output']>;
+  /** the subdirectory you want to run the stack's commands w/in */
+  workdir?: Maybe<Scalars['String']['output']>;
   writeBindings?: Maybe<Array<Maybe<PolicyBinding>>>;
 };
 
@@ -6591,6 +6593,8 @@ export type StackAttributes = {
   git: GitRefAttributes;
   /** optional k8s job configuration for the job that will apply this stack */
   jobSpec?: InputMaybe<GateJobAttributes>;
+  /** whether you want Plural to manage your terraform state for this stack */
+  manageState?: InputMaybe<Scalars['Boolean']['input']>;
   /** the name of the stack */
   name: Scalars['String']['input'];
   observableMetrics?: InputMaybe<Array<InputMaybe<ObservableMetricAttributes>>>;
@@ -6599,6 +6603,8 @@ export type StackAttributes = {
   repositoryId: Scalars['ID']['input'];
   /** A type for the stack, specifies the tool to use to apply it */
   type: StackType;
+  /** the subdirectory you want to run the stack's commands w/in */
+  workdir?: InputMaybe<Scalars['String']['input']>;
   writeBindings?: InputMaybe<Array<InputMaybe<PolicyBindingAttributes>>>;
 };
 

lib/console.ex

@@ -20,6 +20,7 @@ defmodule Console do
 
   @chars String.codepoints("abcdefghijklmnopqrstuvwxyz0123456789")
 
+  def authed_user("deploy-" <> _ = deploy), do: Console.Deployments.Clusters.get_by_deploy_token(deploy)
   def authed_user("console-" <> _ = access), do: Console.Services.Users.get_by_token(access)
   def authed_user(jwt) do
     case Console.Guardian.resource_from_token(jwt) do

lib/console/deployments/policies/policies.ex

@@ -13,7 +13,9 @@ defmodule Console.Deployments.Policies do
     ManagedNamespace,
     StackRun,
     RunStep,
-    RunLog
+    RunLog,
+    Stack,
+    TerraformState
   }
 
   def can?(%User{scopes: [_ | _] = scopes, api: api} = user, res, action) do
@@ -47,6 +49,10 @@ defmodule Console.Deployments.Policies do
     end
   end
 
+  def can?(%Cluster{id: id}, %Stack{cluster_id: id}, :state), do: :pass
+
+  def can?(%User{} = user, %Stack{} = stack, :state), do: can?(user, stack, :write)
+
   def can?(%Cluster{id: id}, %StackRun{cluster_id: id}, _), do: :pass
 
   def can?(%Cluster{} = cluster, %RunStep{} = step, action) do
@@ -77,6 +83,11 @@ defmodule Console.Deployments.Policies do
   def can?(%Cluster{id: id}, %Service{cluster_id: id}, :secrets), do: :pass
   def can?(%Cluster{id: id}, %Service{cluster_id: id}, :read), do: :pass
 
+  def can?(actor, %TerraformState{} = state, :state) do
+    %{stack: stack} = Repo.preload(state, [:stack])
+    can?(actor, stack, :state)
+  end
+
   def can?(%User{} = user, %ClusterBackup{cluster: %Cluster{} = cluster}, action),
     do: can?(user, cluster, action)
 

lib/console/deployments/stacks/state.ex

@@ -0,0 +1,49 @@
+defmodule Console.Deployments.Stacks.State do
+  use Console.Services.Base
+  import Console.Deployments.Policies
+  alias Console.Schema.TerraformState
+
+  def get_tf(stack_id), do: Repo.get_by(TerraformState, stack_id: stack_id)
+
+  def get_terraform_state(stack_id, actor) do
+    case get_tf(stack_id) do
+      %TerraformState{} = state -> allow(state, actor, :state)
+      nil ->
+        stack = Console.Deployments.Stacks.get_stack!(stack_id)
+        with {:ok, _} <- allow(stack, actor, :state),
+          do: {:ok, nil}
+    end
+  end
+
+  def update_terraform_state(state, stack_id, actor) do
+    case get_tf(stack_id) do
+      %TerraformState{} = state -> state
+      _ -> %TerraformState{stack_id: stack_id}
+    end
+    |> TerraformState.changeset(%{state: state})
+    |> allow(actor, :state)
+    |> when_ok(&Repo.insert_or_update/1)
+  end
+
+  def lock_terraform_state(%{"id" => id} = lock_info, stack_id, actor) do
+    case get_tf(stack_id) do
+      nil -> {:error, :not_found}
+      %TerraformState{lock: %{id: ^id}} = state -> set_lock(lock_info, state, actor)
+      %TerraformState{lock: %{id: _} = lock} -> {:error, {:locked, lock}}
+      state -> set_lock(lock_info, state, actor)
+    end
+  end
+
+  def unlock_terraform_state(stack_id, actor) do
+    get_tf(stack_id)
+    |> TerraformState.changeset(%{lock: nil})
+    |> allow(actor, :state)
+    |> when_ok(:update)
+  end
+
+  defp set_lock(lock_info, state, actor) do
+    TerraformState.changeset(state, %{lock: lock_info})
+    |> allow(actor, :state)
+    |> when_ok(:update)
+  end
+end

lib/console/graphql/deployments/stack.ex

@@ -17,6 +17,8 @@ defmodule Console.GraphQl.Deployments.Stack do
     field :job_spec,       :gate_job_attributes, description: "optional k8s job configuration for the job that will apply this stack"
     field :configuration,  non_null(:stack_configuration_attributes), description: "version/image config for the tool you're using"
     field :approval,       :boolean, description: "whether to require approval"
+    field :manage_state,   :boolean, description: "whether you want Plural to manage your terraform state for this stack"
+    field :workdir,        :string, description: "the subdirectory you want to run the stack's commands w/in"
 
     field :read_bindings,  list_of(:policy_binding_attributes)
     field :write_bindings, list_of(:policy_binding_attributes)
@@ -96,6 +98,7 @@ defmodule Console.GraphQl.Deployments.Stack do
     field :approval,            :boolean, description: "whether to require approval"
     field :deleted_at,          :datetime, description: "whether this stack was previously deleted and is pending cleanup"
     field :cancellation_reason, :string, description: "why this run was cancelled"
+    field :workdir,             :string, description: "the subdirectory you want to run the stack's commands w/in"
 
     connection field :runs, node_type: :stack_run do
       arg :pull_request_id, :id

lib/console/schema/stack.ex

@@ -66,6 +66,8 @@ defmodule Console.Schema.Stack do
     field :sha,             :string
     field :last_successful, :string
     field :deleted_at,      :utc_datetime_usec
+    field :manage_state,    :boolean, default: false
+    field :workdir,         :string
 
     field :write_policy_id,  :binary_id
     field :read_policy_id,   :binary_id
@@ -127,7 +129,7 @@ defmodule Console.Schema.Stack do
 
   def stream(query \\ __MODULE__), do: ordered(query, asc: :id)
 
-  @valid ~w(name type paused status approval connection_id repository_id cluster_id)a
+  @valid ~w(name type paused workdir manage_state status approval connection_id repository_id cluster_id)a
 
   def changeset(model, attrs \\ %{}) do
     model

lib/console/schema/terraform_state.ex

@@ -0,0 +1,33 @@
+defmodule Console.Schema.TerraformState do
+  use Piazza.Ecto.Schema
+  alias Console.Schema.Stack
+
+  schema "terraform_states" do
+    field :state, :binary
+
+    embeds_one :lock, Lock, on_replace: :update, primary_key: false do
+      field :id,        :string
+      field :operation, :string
+      field :info,      :string
+      field :who,       :string
+      field :version,   :string
+      field :created,   :string
+      field :path,      :string
+    end
+
+    belongs_to :stack, Stack
+
+    timestamps()
+  end
+
+  def changeset(model, attrs \\ %{}) do
+    model
+    |> cast(attrs, ~w(state stack_id)a)
+    |> cast_embed(:lock, with: &lock_changeset/2)
+    |> unique_constraint(:stack_id)
+  end
+
+  def lock_changeset(model, attrs) do
+    cast(model, attrs, ~w(id operation info who version created path)a)
+  end
+end

lib/console_web/controllers/stack_controller.ex

@@ -0,0 +1,51 @@
+defmodule ConsoleWeb.StackController do
+  use ConsoleWeb, :controller
+  alias Console.Deployments.Stacks.State
+
+  plug :get_actor when action in [:get_tf_state, :update_tf_state, :lock_tf_state, :unlock_tf_state]
+
+  def get_tf_state(conn, %{"stack_id" => id}) do
+    case State.get_terraform_state(id, conn.assigns.actor) do
+      {:ok, %{state: state}} -> send_resp(conn, 200, state)
+      {:ok, nil} -> send_resp(conn, 204, "")
+      _ -> send_resp(conn, 403, "Forbidden")
+    end
+  end
+
+  def update_tf_state(conn, %{"stack_id" => id}) do
+    %{raw_body: state, actor: actor} = conn.assigns
+
+    IO.iodata_to_binary(state)
+    |> State.update_terraform_state(id, actor)
+    |> handle_resp(conn)
+  end
+
+  def lock_tf_state(conn, %{"stack_id" => id} = lock) do
+    State.lock_terraform_state(lock, id, conn.assigns.actor)
+    |> handle_resp(conn)
+  end
+
+  def unlock_tf_state(conn, %{"stack_id" => id}) do
+    State.unlock_terraform_state(id, conn.assigns.actor)
+    |> handle_resp(conn)
+  end
+
+  defp handle_resp({:error, {:locked, lock}}, conn) do
+    put_resp_header(conn, "content-type", "application/json")
+    |> send_resp(423, Poison.encode!(Map.from_struct(lock)))
+  end
+
+  defp handle_resp({:error, :forbidden}, conn), do: send_resp(conn, 403, "Forbidden")
+  defp handle_resp({:error, err}, conn), do: send_resp(conn, 400, inspect(err))
+
+  defp handle_resp({:ok, _}, conn), do: send_resp(conn, 200, "")
+
+  defp get_actor(conn, _) do
+    with {_, token} <- Plug.BasicAuth.parse_basic_auth(conn),
+         %{} = actor <- Console.authed_user(token) do
+      assign(conn, :actor, actor)
+    else
+      _ -> send_resp(conn, 403, "Forbidden") |> halt()
+    end
+  end
+end

lib/console_web/router.ex

@@ -28,6 +28,15 @@ defmodule ConsoleWeb.Router do
 
       scope "/v1", ConsoleWeb do
         post "/webhooks/:type/:id", WebhookController, :scm
+
+        scope "/states" do
+          scope "/terraform" do
+            get "/:stack_id", StackController, :get_tf_state
+            post "/:stack_id", StackController, :update_tf_state
+            post "/:stack_id/lock", StackController, :lock_tf_state
+            post "/:stack_id/unlock", StackController, :unlock_tf_state
+          end
+        end
       end
     end
 

priv/repo/migrations/20240526120358_add_terraform_state.exs

@@ -0,0 +1,20 @@
+defmodule Console.Repo.Migrations.AddTerraformState do
+  use Ecto.Migration
+
+  def change do
+    create table(:terraform_states, primary_key: false) do
+      add :id,       :uuid, primary_key: true
+      add :stack_id, references(:stacks, type: :uuid, on_delete: :delete_all)
+      add :state,    :binary
+      add :lock,     :map
+
+      timestamps()
+    end
+
+    alter table(:stacks) do
+      add :manage_state, :boolean, default: false
+    end
+
+    create unique_index(:terraform_states, [:stack_id])
+  end
+end

priv/repo/migrations/20240526201015_add_workdir.exs

@@ -0,0 +1,9 @@
+defmodule Console.Repo.Migrations.AddWorkdir do
+  use Ecto.Migration
+
+  def change do
+    alter table(:stacks) do
+      add :workdir, :string
+    end
+  end
+end

schema/schema.graphql

@@ -984,6 +984,12 @@ input StackAttributes {
   "whether to require approval"
   approval: Boolean
 
+  "whether you want Plural to manage your terraform state for this stack"
+  manageState: Boolean
+
+  "the subdirectory you want to run the stack's commands w\/in"
+  workdir: String
+
   readBindings: [PolicyBindingAttributes]
 
   writeBindings: [PolicyBindingAttributes]
@@ -1114,6 +1120,9 @@ type InfrastructureStack {
   "why this run was cancelled"
   cancellationReason: String
 
+  "the subdirectory you want to run the stack's commands w\/in"
+  workdir: String
+
   runs(after: String, first: Int, before: String, last: Int, pullRequestId: ID): StackRunConnection
 
   pullRequests(after: String, first: Int, before: String, last: Int): PullRequestConnection