Merge pull request #16 from appunite/replace-porcelain-with-erlexec

Replace Porcelain with erlexec

.circleci/config.yml

@@ -1,18 +1,12 @@
 # Elixir CircleCI 2.0 configuration file
 #
 # Check https://circleci.com/docs/2.0/language-elixir/ for more details
-version: 2
-jobs:
-  test:
-    docker:
-      # specify the version here
-      - image: appunite/elixir-ci:1.7.1
+version: 2.1
 
-    working_directory: ~/repo
+commands:
+  fetch_deps:
+    description: "Fetch deps, build, and cache them"
     steps:
-      - run: apt-get update && apt-get install -y --no-install-recommends imagemagick ghostscript
-      - checkout
-
       - restore_cache:
           keys:
             - deps-v1-{{ .Branch }}-{{ .Revision }}
@@ -25,8 +19,23 @@ jobs:
             - "deps"
             - "_build"
 
+jobs:
+  test:
+    docker:
+      # specify the version here
+      - image: appunite/elixir-ci:1.7.1
+
+    working_directory: /var/repo
+    steps:
+      - run: apt-get update && apt-get install -y --no-install-recommends sudo imagemagick ghostscript
+      - checkout
+      - fetch_deps
+
       - run: mkdir -p reports/exunit
-      - run: mix coveralls.circle --exclude exec:true
+      - run:
+          command: mix coveralls.circle
+          environment:
+            IMAGER_USER: nobody
       - store_test_results:
           path: reports
   dialyzer:
@@ -35,18 +44,7 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-
-      - restore_cache:
-          keys:
-            - deps-v1-{{ .Branch }}-{{ .Revision }}
-            - deps-v1-{{ .Branch }}-
-            - deps-v1-
-      - run: mix do deps.get, deps.compile
-      - save_cache:
-          key: deps-v1-{{ .Branch }}-{{ .Revision }}
-          paths:
-            - "deps"
-            - "_build"
+      - fetch_deps
 
       - restore_cache:
           keys:
@@ -66,20 +64,9 @@ jobs:
     working_directory: ~/repo
     steps:
       - checkout
-
-      - restore_cache:
-          keys:
-            - deps-v1-{{ .Branch }}-{{ .Revision }}
-            - deps-v1-{{ .Branch }}-
-            - deps-v1-
-      - run: mix do deps.get, deps.compile
-      - save_cache:
-          key: deps-v1-{{ .Branch }}-{{ .Revision }}
-          paths:
-            - "deps"
-            - "_build"
-
+      - fetch_deps
       - run: mix format --check-formatted
+
 workflows:
   version: 2
   testing:

Dockerfile

@@ -1,22 +1,19 @@
-FROM alpine:latest AS goon
-RUN wget -O goon.tar.gz https://github.com/alco/goon/releases/download/v1.1.1/goon_linux_amd64.tar.gz \
-    && gzip -d goon.tar.gz \
-    && tar xf goon.tar
-
 FROM ubuntu:latest AS source
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl1.1 imagemagick ghostscript \
+    libssl1.1 imagemagick ghostscript libcap \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=goon /goon /usr/bin/goon
 ENV LANG C.UTF-8
 ENV PORT 80
 
 FROM appunite/elixir-ci:1.7.1 AS build
 ENV MIX_ENV prod
+ENV OPTIMIZE true
 COPY . /app
 WORKDIR /app
+RUN apt-get install libcap-dev
 RUN mix local.hex --force && mix local.rebar --force
-RUN mix deps.get && mix compile && mix release --env=prod
+RUN mix deps.get
+RUN mix compile && mix release --env=prod
 
 FROM source
 MAINTAINER Łukasz Jan Niemier <lukasz.niemier@appunite.com>

config/dev.exs

@@ -20,7 +20,7 @@ config :logger, :console, format: "[$level] $message\n"
 config :imager, :stores, %{
   "local" => %{
     store: {Imager.Store.Local, dir: "test/fixtures/"},
-    cache: {Imager.Store.Local, dir: "tmp/cache/"}
+    cache: {Imager.Store.Blackhole, []}
   }
 }
 

config/test.exs

@@ -6,6 +6,8 @@ config :imager, ImagerWeb.Endpoint,
   server: false,
   instrumenters: []
 
+config :imager, :user, System.get_env("IMAGER_USER")
+
 config :imager, :port, 4001
 
 config :logger, :console, format: "[$level] $message\n"

lib/imager.ex

@@ -7,6 +7,8 @@ defmodule Imager do
   if it comes from the database, an external API or others.
   """
 
+  import Mockery.Macro
+
   require Logger
 
   alias Imager.Stats
@@ -38,32 +40,29 @@ defmodule Imager do
     ))
 
     Logger.metadata(input: file_name, commands: inspect(commands))
+    Logger.debug(inspect(args))
 
     with :error <- Store.retrieve(cache, result_name, opts),
-         {:ok, {_, _, stream}} <- Store.retrieve(store, file_name, opts) do
+         {:ok, {_, _, in_stream}} <- Store.retrieve(store, file_name, opts) do
       Logger.debug("Start processing")
       Stats.increment("imager.process.started", 1, tags: tags)
 
-      process =
-        Porcelain.spawn(executable(), args,
-          in: stream,
-          out: :stream,
-          err: :string
-        )
+      {pid, out_stream} = runner().stream(executable(), args)
+
+      runner().feed_stream(pid, in_stream)
 
-      case Porcelain.Process.await(process) do
-        {:ok, %{status: 0}} ->
+      case runner().wait(pid) do
+        :ok ->
           Stats.increment("imager.process.succeeded", 1, tags: tags)
 
           stream =
-            process.out
+            out_stream
             |> Store.store(cache, mime, result_name, opts)
 
           {:ok, {:unknown, mime, stream}}
 
         _ ->
           Stats.increment("imager.process.failed", 1, tags: tags)
-          Logger.error(process.err)
 
           :failed
       end
@@ -91,5 +90,7 @@ defmodule Imager do
     end
   end
 
+  defp runner, do: mockable(Imager.Runner)
+
   defp executable, do: Application.get_env(:imager, :executable, "convert")
 end

lib/imager/application.ex

@@ -5,6 +5,8 @@ defmodule Imager.Application do
 
   def start(_type, _args) do
     children = [
+      exec_app(),
+      {DynamicSupervisor, name: Imager.Workers, strategy: :one_for_one},
       ImagerWeb.Endpoint
     ]
 
@@ -26,4 +28,16 @@ defmodule Imager.Application do
 
     :ok
   end
+
+  defp exec_app do
+    opts =
+      with {:ok, name} when not is_nil(name) <-
+             Application.fetch_env(:imager, :user) do
+        [user: String.to_charlist(name)]
+      else
+        _ -> []
+      end
+
+    %{id: :exec_app, start: {:exec, :start_link, [opts]}}
+  end
 end

lib/imager/runner.ex

@@ -0,0 +1,105 @@
+defmodule Imager.Runner do
+  use GenServer, restart: :temporary
+
+  require Logger
+
+  defstruct [:output, :pid, :ospid]
+
+  def stream(cmd, args \\ []) do
+    {:ok, pid} =
+      DynamicSupervisor.start_child(
+        Imager.Workers,
+        {__MODULE__, pid: self(), cmd: cmd, args: args}
+      )
+
+    stream =
+      Stream.resource(
+        fn -> pid end,
+        fn pid ->
+          receive do
+            {:out, ^pid, data} ->
+              {[data], pid}
+
+            {:exit, ^pid, status} = msg ->
+              send(self(), msg)
+
+              {:halt, status}
+          end
+        end,
+        fn _ -> nil end
+      )
+
+    {pid, stream}
+  end
+
+  def start_link(args), do: GenServer.start_link(__MODULE__, args)
+
+  def wait(pid) do
+    receive do
+      {:exit, ^pid, :success} -> :ok
+      {:exit, ^pid, _} -> :error
+    end
+  end
+
+  def feed_stream(pid, stream) do
+    Enum.each(stream, &feed(pid, &1))
+
+    feed(pid, :eof)
+  end
+
+  def feed(pid, data) when is_binary(data) or data == :eof,
+    do: GenServer.call(pid, {:in, data})
+
+  def init(opts) do
+    output = Keyword.fetch!(opts, :pid)
+    cmd = Keyword.fetch!(opts, :cmd)
+    cmd = System.find_executable(cmd) || cmd
+    args = Keyword.get(opts, :args, [])
+    exec = Enum.map([cmd | args], &String.to_charlist/1)
+
+    {:ok, pid, ospid} = :exec.run(exec, [:stdin, :stdout, :stderr, :monitor])
+
+    Process.sleep(100)
+
+    {:ok, %__MODULE__{output: output, pid: pid, ospid: ospid}}
+  end
+
+  def handle_call({:in, data}, _ref, %__MODULE__{ospid: pid} = state) do
+    {:reply, :exec.send(pid, data), state}
+  end
+
+  def handle_info(
+        {:stdout, ospid, data},
+        %__MODULE__{output: output, ospid: ospid} = state
+      ) do
+    send(output, {:out, self(), data})
+
+    {:noreply, state}
+  end
+
+  def handle_info(
+        {:stderr, ospid, data},
+        %__MODULE__{ospid: ospid} = state
+      ) do
+    Logger.warn(inspect(data))
+
+    {:noreply, state}
+  end
+
+  def handle_info(
+        {:DOWN, ospid, :process, pid, result},
+        %__MODULE__{pid: pid, ospid: ospid} = state
+      ) do
+    handle_result(result, state)
+
+    {:stop, :normal, state}
+  end
+
+  defp handle_result(:normal, %__MODULE__{output: output}),
+    do: send(output, {:exit, self(), :success})
+
+  defp handle_result({:exit_status, status}, %__MODULE__{output: output}) do
+    Logger.warn(inspect(:exec.status(status)))
+    send(output, {:exit, self(), :failure})
+  end
+end

lib/imager/store/local.ex

@@ -20,7 +20,7 @@ defmodule Imager.Store.Local do
     mime = MIME.type(extname)
 
     with {:ok, %File.Stat{size: size}} <- File.stat(full_path) do
-      {:ok, {size, mime, File.stream!(full_path, [], 1024 * 1024)}}
+      {:ok, {size, mime, File.stream!(full_path, [], 2 * 1024)}}
     else
       _ -> :error
     end

lib/imager/store/s3.ex

@@ -12,7 +12,7 @@ defmodule Imager.Store.S3 do
 
   def retrieve(path, opts) do
     {bucket, config} = Keyword.pop(opts, :bucket)
-    {chunk_size, config} = Keyword.pop(config, :chunk_size, 1024 * 1024)
+    {chunk_size, config} = Keyword.pop(config, :chunk_size, 2 * 1024)
 
     with {:ok, size, mime} <- get_file_size(bucket, path, config) do
       stream =

lib/imager_web/controllers/image.ex

@@ -6,6 +6,7 @@ defmodule ImagerWeb.Controllers.Image do
   """
 
   import Plug.Conn
+  import Mockery.Macro
 
   require Logger
 
@@ -21,12 +22,12 @@ defmodule ImagerWeb.Controllers.Image do
 
     Logger.metadata(path: path, commands: inspect(commands))
 
-    with {:ok, store} <- Imager.store(store),
-         {:ok, {size, mime, stream}} <- Imager.process(store, path, commands) do
+    with {:ok, store} <- imager().store(store),
+         {:ok, {size, mime, stream}} <- imager().process(store, path, commands) do
       conn =
         conn
-        |> put_resp_content_type(mime)
-        |> put_resp_content_size(size)
+        |> put_resp_content_type(mime, nil)
+        |> put_resp_content_length(size)
         |> send_chunked(200)
 
       Enum.reduce_while(stream, conn, fn chunk, conn ->
@@ -46,8 +47,10 @@ defmodule ImagerWeb.Controllers.Image do
 
   def get(conn, _), do: send_resp(conn, 404, "")
 
-  defp put_resp_content_size(conn, :unknown), do: conn
+  defp imager, do: mockable(Imager)
 
-  defp put_resp_content_size(conn, size) when is_integer(size),
-    do: put_resp_header(conn, "content-size", Integer.to_string(size))
+  defp put_resp_content_length(conn, :unknown), do: conn
+
+  defp put_resp_content_length(conn, size) when is_integer(size),
+    do: put_resp_header(conn, "content-length", Integer.to_string(size))
 end