diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex
index 712676637e5..3b1578249c5 100644
--- a/lib/mix/lib/mix/compilers/elixir.ex
+++ b/lib/mix/lib/mix/compilers/elixir.ex
@@ -29,7 +29,7 @@ defmodule Mix.Compilers.Elixir do
   """
   def compile(manifest, srcs, dest, force, opts) do
     all = Mix.Utils.extract_files(srcs, [:ex])
-    {all_modules, all_sources} = parse_manifest(manifest)
+    {all_modules, all_sources} = parse_manifest(manifest, dest)
     modified = Mix.Utils.last_modified(manifest)
 
     removed =
@@ -73,7 +73,7 @@ defmodule Mix.Compilers.Elixir do
         compile_manifest(manifest, modules, sources, stale, dest, opts)
         :ok
       removed != [] ->
-        write_manifest(manifest, modules, sources)
+        write_manifest(manifest, modules, sources, dest)
         :ok
       true ->
         :noop
@@ -91,20 +91,20 @@ defmodule Mix.Compilers.Elixir do
   @doc """
   Removes compiled files for the given `manifest`.
   """
-  def clean(manifest) do
-    Enum.each read_manifest(manifest), fn
+  def clean(manifest, compile_path) do
+    Enum.each(read_manifest(manifest, compile_path), fn
       module(beam: beam) ->
         File.rm(beam)
       _ ->
         :ok
-    end
+    end)
   end
 
   @doc """
   Returns protocols and implementations for the given `manifest`.
   """
-  def protocols_and_impls(manifest) do
-    for module(beam: beam, module: module, kind: kind) <- read_manifest(manifest),
+  def protocols_and_impls(manifest, compile_path) do
+    for module(beam: beam, module: module, kind: kind) <- read_manifest(manifest, compile_path),
         match?(:protocol, kind) or match?({:impl, _}, kind),
         do: {module, kind, beam}
   end
@@ -112,13 +112,15 @@ defmodule Mix.Compilers.Elixir do
   @doc """
   Reads the manifest.
   """
-  def read_manifest(manifest) do
+  def read_manifest(manifest, compile_path) do
     # TODO: No longer support file consult once v1.3 is out
     try do
       manifest |> File.read!() |> :erlang.binary_to_term()
     else
-      [@manifest_vsn | t] -> t
-      _ -> []
+      [@manifest_vsn | t] ->
+        expand_beam_paths(t, compile_path)
+      _ ->
+        []
     rescue
       _ -> []
     end
@@ -149,12 +151,12 @@ defmodule Mix.Compilers.Elixir do
 
     try do
       _ = Kernel.ParallelCompiler.files stale,
-            [each_module: &each_module(pid, dest, cwd, &1, &2, &3),
+            [each_module: &each_module(pid, cwd, &1, &2, &3),
              each_long_compilation: &each_long_compilation(&1, long_compilation_threshold),
              long_compilation_threshold: long_compilation_threshold,
              dest: dest] ++ extra
       Agent.cast pid, fn {modules, sources} ->
-        write_manifest(manifest, modules, sources)
+        write_manifest(manifest, modules, sources, dest)
         {modules, sources}
       end
     after
@@ -170,11 +172,8 @@ defmodule Mix.Compilers.Elixir do
     |> Code.compiler_options()
   end
 
-  defp each_module(pid, dest, cwd, source, module, binary) do
-    beam =
-      dest
-      |> Path.join(Atom.to_string(module) <> ".beam")
-      |> Path.relative_to(cwd)
+  defp each_module(pid, cwd, source, module, binary) do
+    beam = Atom.to_string(module) <> ".beam"
 
     {compile_references, runtime_references} = Kernel.LexicalTracker.remote_references(module)
 
@@ -338,9 +337,7 @@ defmodule Mix.Compilers.Elixir do
   ## Manifest handling
 
   # Similar to read_manifest, but supports data migration.
-  defp parse_manifest(manifest) do
-    state = {[], []}
-
+  defp parse_manifest(manifest, compile_path) do
     # TODO: No longer support file consult once v1.3 is out
     manifest =
       try do
@@ -351,37 +348,53 @@ defmodule Mix.Compilers.Elixir do
 
     case manifest do
       {:ok, [@manifest_vsn | data]} ->
-        parse_manifest(data, state)
+        do_parse_manifest(data, compile_path)
       {:ok, [:v2 | data]} ->
-        for {beam, module, _, _, _, _, _, _} <- data do
-          remove_and_purge(beam, module)
-        end
-        state
+        for {beam, module, _, _, _, _, _, _} <- data,
+            do: remove_and_purge(beam, module)
+        {[], []}
       _ ->
-        state
+        {[], []}
     end
   end
 
-  defp parse_manifest(data, state) do
-    Enum.reduce data, state, fn
+  defp do_parse_manifest(data, compile_path) do
+    Enum.reduce(data, {[], []}, fn
       module() = module, {modules, sources} ->
-        {[module | modules], sources}
+        {[expand_beam_path(module, compile_path) | modules], sources}
       source() = source, {modules, sources} ->
         {modules, [source | sources]}
-    end
+    end)
+  end
+
+  defp expand_beam_path(module(beam: beam) = module, compile_path) do
+    module(module, beam: Path.join(compile_path, beam))
   end
 
-  defp write_manifest(manifest, [], []) do
+  defp expand_beam_paths(modules, ""), do: modules
+  defp expand_beam_paths(modules, compile_path) do
+    Enum.map(modules, fn
+      module() = module ->
+        expand_beam_path(module, compile_path)
+      other ->
+        other
+    end)
+  end
+
+  defp write_manifest(manifest, [], [], _compile_path) do
     File.rm(manifest)
     :ok
   end
 
-  defp write_manifest(manifest, modules, sources) do
+  defp write_manifest(manifest, modules, sources, compile_path) do
     File.mkdir_p!(Path.dirname(manifest))
 
     modules =
       for module(beam: beam, binary: binary) = module <- modules do
-        if binary, do: File.write!(beam, binary)
+        if binary do
+          beam_path = Path.join(compile_path, beam)
+          File.write!(beam_path, binary)
+        end
         module(module, binary: nil)
       end
 
diff --git a/lib/mix/lib/mix/compilers/test.ex b/lib/mix/lib/mix/compilers/test.ex
index 608abd71291..e8596370602 100644
--- a/lib/mix/lib/mix/compilers/test.ex
+++ b/lib/mix/lib/mix/compilers/test.ex
@@ -21,12 +21,12 @@ defmodule Mix.Compilers.Test do
   It expects all of the test patterns, the test files that were matched for the
   test patterns, the test paths, and the opts from the test task.
   """
-  def require_and_run(test_patterns, matched_test_files, test_paths, opts) do
+  def require_and_run(test_patterns, matched_test_files, test_paths, compile_path, opts) do
     stale = opts[:stale]
 
     {test_files_to_run, stale_manifest_pid, parallel_require_callbacks} =
       if stale do
-        set_up_stale(matched_test_files, test_paths, opts)
+        set_up_stale(matched_test_files, test_paths, compile_path, opts)
       else
         {matched_test_files, nil, []}
       end
@@ -63,7 +63,7 @@ defmodule Mix.Compilers.Test do
     end
   end
 
-  defp set_up_stale(matched_test_files, test_paths, opts) do
+  defp set_up_stale(matched_test_files, test_paths, compile_path, opts) do
     manifest = manifest()
     modified = Mix.Utils.last_modified(manifest)
     all_sources = read_manifest()
@@ -100,7 +100,7 @@ defmodule Mix.Compilers.Test do
 
     test_files_to_run =
       sources
-      |> tests_with_changed_references()
+      |> tests_with_changed_references(compile_path)
       |> MapSet.union(stale)
       |> MapSet.to_list()
 
@@ -197,13 +197,13 @@ defmodule Mix.Compilers.Test do
 
   ## Test changed dependency resolution
 
-  defp tests_with_changed_references(test_sources) do
+  defp tests_with_changed_references(test_sources, compile_path) do
     test_manifest = manifest()
     [elixir_manifest] = Mix.Tasks.Compile.Elixir.manifests()
 
     if Mix.Utils.stale?([elixir_manifest], [test_manifest]) do
       elixir_manifest_entries =
-        CE.read_manifest(elixir_manifest)
+        CE.read_manifest(elixir_manifest, compile_path)
         |> Enum.group_by(&elem(&1, 0))
 
       stale_modules =
diff --git a/lib/mix/lib/mix/tasks/compile.elixir.ex b/lib/mix/lib/mix/tasks/compile.elixir.ex
index 2d2a6c2c653..e92bdbc7ce7 100644
--- a/lib/mix/lib/mix/tasks/compile.elixir.ex
+++ b/lib/mix/lib/mix/tasks/compile.elixir.ex
@@ -71,6 +71,7 @@ defmodule Mix.Tasks.Compile.Elixir do
   Cleans up compilation artifacts.
   """
   def clean do
-    Mix.Compilers.Elixir.clean(manifest())
+    dest = Mix.Project.compile_path
+    Mix.Compilers.Elixir.clean(manifest(), dest)
   end
 end
diff --git a/lib/mix/lib/mix/tasks/compile.protocols.ex b/lib/mix/lib/mix/tasks/compile.protocols.ex
index 29e9a2d0252..e2b57465a33 100644
--- a/lib/mix/lib/mix/tasks/compile.protocols.ex
+++ b/lib/mix/lib/mix/tasks/compile.protocols.ex
@@ -88,8 +88,9 @@ defmodule Mix.Tasks.Compile.Protocols do
 
     protocols_and_impls =
       for path <- [app | deps] do
-        elixir = Path.join(path, ".compile.elixir")
-        Mix.Compilers.Elixir.protocols_and_impls(elixir)
+        manifest_path = Path.join(path, ".compile.elixir")
+        compile_path = Path.join(path, "ebin")
+        Mix.Compilers.Elixir.protocols_and_impls(manifest_path, compile_path)
       end
 
     Enum.concat(protocols_and_impls)
diff --git a/lib/mix/lib/mix/tasks/test.ex b/lib/mix/lib/mix/tasks/test.ex
index 7ab2fbfd714..e7c807f9bb4 100644
--- a/lib/mix/lib/mix/tasks/test.ex
+++ b/lib/mix/lib/mix/tasks/test.ex
@@ -184,12 +184,13 @@ defmodule Mix.Tasks.Test do
     end
 
     project = Mix.Project.config
+    compile_path = Mix.Project.compile_path(project)
 
     # Start cover after we load deps but before we start the app.
     cover =
       if opts[:cover] do
         cover = Keyword.merge(@cover, project[:test_coverage] || [])
-        cover[:tool].start(Mix.Project.compile_path(project), cover)
+        cover[:tool].start(compile_path, cover)
       end
 
     # Start the app and configure exunit with command line options
@@ -226,7 +227,7 @@ defmodule Mix.Tasks.Test do
 
     display_warn_test_pattern(matched_warn_test_files, test_pattern)
 
-    case CT.require_and_run(files, matched_test_files, test_paths, opts) do
+    case CT.require_and_run(files, matched_test_files, test_paths, compile_path, opts) do
       {:ok, %{failures: failures}} ->
         cover && cover.()
 
diff --git a/lib/mix/lib/mix/tasks/xref.ex b/lib/mix/lib/mix/tasks/xref.ex
index 0df28ad2006..a21164da45f 100644
--- a/lib/mix/lib/mix/tasks/xref.ex
+++ b/lib/mix/lib/mix/tasks/xref.ex
@@ -2,7 +2,7 @@ defmodule Mix.Tasks.Xref do
   use Mix.Task
 
   alias Mix.Tasks.Compile.Elixir, as: E
-  import Mix.Compilers.Elixir, only: [read_manifest: 1, source: 1, source: 2]
+  import Mix.Compilers.Elixir, only: [read_manifest: 2, source: 1, source: 2]
 
   @shortdoc "Performs cross reference checks"
   @recursive true
@@ -289,7 +289,7 @@ defmodule Mix.Tasks.Xref do
 
   defp each_source_entries(entries_fun, pair_fun) do
     for manifest <- E.manifests(),
-        source(source: file) = source <- read_manifest(manifest),
+        source(source: file) = source <- read_manifest(manifest, ""),
         entries = entries_fun.(source),
         entries != [] and entries != %{},
         do: pair_fun.(file, entries)