[WIP] Copy file mode local push

dvc/utils/fs.py

@@ -189,7 +189,8 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     dest = fspath_py35(dest)
 
     name = name if name else os.path.basename(dest)
-    total = os.stat(src).st_size
+    src_stat = os.stat(src)
+    total = src_stat.st_size
 
     if os.path.isdir(dest):
         dest = os.path.join(dest, os.path.basename(src))
@@ -211,6 +212,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
                     if not buf:
                         break
                     fdest_wrapped.write(buf)
+        os.chmod(dest, src_stat.st_mode)
 
 
 def walk_files(directory):

tests/dir_helpers.py

@@ -60,7 +60,6 @@
     "tmp_dir",
     "scm",
     "dvc",
-    "local_remote",
     "run_copy",
     "erepo_dir",
     "git_dir",
@@ -200,6 +199,31 @@ def branch(self, name, new=False):
         finally:
             self.scm.checkout(old)
 
+    @property
+    def local_remote_path(self):
+        return self.parent / (self.name + "_local_remote")
+
+    @contextmanager
+    def local_remote_context(self):
+        self._require("dvc")
+        self.setup_remote()
+        with self.chdir():
+            yield
+            self.dvc.push()
+
+    def setup_remote(self, remote_url=None):
+        self._require("dvc")
+
+        if not remote_url:
+            # local by default
+            remote_url = fspath(self.local_remote_path)
+        with self.dvc.config.edit() as conf:
+            conf["remote"]["upstream"] = {"url": remote_url}
+            conf["core"]["remote"] = "upstream"
+        if hasattr(self, "scm"):
+            self.scm_add([self.dvc.config.files["repo"]], commit="add remote")
+        return remote_url
+
 
 def _coerce_filenames(filenames):
     if isinstance(filenames, (str, bytes, pathlib.PurePath)):
@@ -257,17 +281,6 @@ def _git_init(path):
     git.close()
 
 
-@pytest.fixture
-def local_remote(request, tmp_dir, dvc, make_tmp_dir):
-    path = make_tmp_dir("local-remote")
-    with dvc.config.edit() as conf:
-        conf["remote"]["upstream"] = {"url": fspath(path)}
-        conf["core"]["remote"] = "upstream"
-    if "scm" in request.fixturenames:
-        tmp_dir.scm_add([dvc.config.files["repo"]], commit="add remote")
-    return path
-
-
 @pytest.fixture
 def run_copy(tmp_dir, dvc):
     tmp_dir.gen(
@@ -288,17 +301,7 @@ def run_copy(src, dst, **run_kwargs):
 
 @pytest.fixture
 def erepo_dir(make_tmp_dir):
-    path = make_tmp_dir("erepo", scm=True, dvc=True)
-
-    # Chdir for git and dvc to work locally
-    with path.chdir():
-        with path.dvc.config.edit() as conf:
-            cache_dir = path.dvc.cache.local.cache_dir
-            conf["remote"]["upstream"] = {"url": cache_dir}
-            conf["core"]["remote"] = "upstream"
-        path.scm_add([path.dvc.config.files["repo"]], commit="add remote")
-
-    return path
+    return make_tmp_dir("erepo", scm=True, dvc=True)
 
 
 @pytest.fixture

tests/func/test_api.py

@@ -39,7 +39,7 @@ def test_get_url(tmp_dir, dvc, remote_url):
 
 @pytest.mark.parametrize("remote_url", remote_params, indirect=True)
 def test_get_url_external(erepo_dir, remote_url):
-    _set_remote_url_and_commit(erepo_dir.dvc, remote_url)
+    erepo_dir.setup_remote(remote_url)
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "foo", commit="add foo")
 
@@ -74,7 +74,7 @@ def test_open(remote_url, tmp_dir, dvc):
 
 @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
 def test_open_external(remote_url, erepo_dir):
-    _set_remote_url_and_commit(erepo_dir.dvc, remote_url)
+    erepo_dir.setup_remote(remote_url)
 
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("version", "master", commit="add version")
@@ -108,13 +108,6 @@ def test_missing(remote_url, tmp_dir, dvc):
         api.read("foo")
 
 
-def _set_remote_url_and_commit(repo, remote_url):
-    with repo.config.edit() as conf:
-        conf["remote"]["upstream"]["url"] = remote_url
-    repo.scm.add([repo.config.files["repo"]])
-    repo.scm.commit("modify remote")
-
-
 def test_open_scm_controlled(tmp_dir, erepo_dir):
     erepo_dir.scm_gen({"scm_controlled": "file content"}, commit="create file")
 

tests/func/test_data_cloud.py

@@ -6,6 +6,7 @@
 from unittest import SkipTest
 
 import pytest
+from funcy import first
 
 from dvc.compat import fspath, fspath_py35
 from dvc.cache import NamedCache
@@ -688,14 +689,11 @@ def test(self):
 
 
 def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
-    tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
-    tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")
-
-    dvc.config["remote"]["local_remote"] = {
-        "url": fspath(tmp_path_factory.mktemp("local_remote"))
-    }
-    dvc.config["core"]["remote"] = "local_remote"
-    dvc.push()
+    with tmp_dir.local_remote_context():
+        tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
+        tmp_dir.dvc_gen(
+            {"dir": {"subfile": "file2 content"}}, commit="add dir"
+        )
 
     # remove artifacts and cache to trigger fetching
     remove("file")
@@ -710,7 +708,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
     # Removing cache will invalidate existing state entries
     remove(dvc.cache.local.cache_dir)
 
-    dvc.config["remote"]["local_remote"]["verify"] = True
+    dvc.config["remote"]["upstream"]["verify"] = True
 
     dvc.pull()
     assert checksum_spy.call_count == 3
@@ -783,13 +781,13 @@ def recurse_list_dir(d):
     ]
 
 
-def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy):
-    (stage0,) = tmp_dir.dvc_gen("foo", "foo")
-    stage1 = run_copy("foo", "bar", single_stage=True)
-    stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
-    outs = ["foo", "bar", "foobar"]
+def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy):
+    with tmp_dir.local_remote_context():
+        (stage0,) = tmp_dir.dvc_gen("foo", "foo")
+        stage1 = run_copy("foo", "bar", single_stage=True)
+        stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
+        outs = ["foo", "bar", "foobar"]
 
-    dvc.push()
     clean(outs, dvc)
     dvc.pull()
     assert all((tmp_dir / file).exists() for file in outs)
@@ -813,21 +811,21 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy):
     assert set(stats["added"]) == set(outs)
 
 
-def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
-    tmp_dir.dvc_gen("foo", "foo")
-    run_copy("foo", "bar", single_stage=True)
+def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy):
+    with tmp_dir.local_remote_context():
+        tmp_dir.dvc_gen("foo", "foo")
+        run_copy("foo", "bar", single_stage=True)
 
-    tmp_dir.dvc_gen("lorem", "lorem")
-    run_copy("lorem", "lorem2", name="copy-lorem-lorem2")
+        tmp_dir.dvc_gen("lorem", "lorem")
+        run_copy("lorem", "lorem2", name="copy-lorem-lorem2")
 
-    tmp_dir.dvc_gen("ipsum", "ipsum")
-    run_copy("ipsum", "baz", name="copy-ipsum-baz")
+        tmp_dir.dvc_gen("ipsum", "ipsum")
+        run_copy("ipsum", "baz", name="copy-ipsum-baz")
 
     outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
 
-    dvc.push()
     # each one's a copy of other, hence 3
-    assert len(recurse_list_dir(fspath_py35(local_remote))) == 3
+    assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 3
 
     clean(outs, dvc)
     assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
@@ -836,13 +834,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
     assert set(dvc.pull()["added"]) == set(outs)
 
     # clean everything in remote and push
-    clean(local_remote.iterdir())
+    clean(tmp_dir.local_remote_path.iterdir())
     dvc.push(["dvc.yaml:copy-ipsum-baz"])
-    assert len(recurse_list_dir(fspath_py35(local_remote))) == 1
+    assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 1
 
-    clean(local_remote.iterdir())
+    clean(tmp_dir.local_remote_path.iterdir())
     dvc.push(["dvc.yaml"])
-    assert len(recurse_list_dir(fspath_py35(local_remote))) == 2
+    assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 2
 
     with pytest.raises(StageNotFound):
         dvc.push(["dvc.yaml:StageThatDoesNotExist"])
@@ -859,8 +857,10 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
         ({}, "Everything is up to date"),
     ],
 )
-def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
+def test_push_stats(tmp_dir, dvc, fs, msg, caplog):
+    tmp_dir.setup_remote()
     tmp_dir.dvc_gen(fs)
+
     caplog.clear()
     with caplog.at_level(level=logging.INFO, logger="dvc"):
         main(["push"])
@@ -875,19 +875,19 @@ def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
         ({}, "Everything is up to date."),
     ],
 )
-def test_fetch_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
-    tmp_dir.dvc_gen(fs)
-    dvc.push()
+def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog):
+    with tmp_dir.local_remote_context():
+        tmp_dir.dvc_gen(fs)
     clean(list(fs.keys()), dvc)
     caplog.clear()
     with caplog.at_level(level=logging.INFO, logger="dvc"):
         main(["fetch"])
     assert msg in caplog.text
 
 
-def test_pull_stats(tmp_dir, dvc, local_remote, caplog):
-    tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
-    dvc.push()
+def test_pull_stats(tmp_dir, dvc, caplog):
+    with tmp_dir.local_remote_context():
+        tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
     clean(["foo", "bar"], dvc)
     (tmp_dir / "bar").write_text("foobar")
     caplog.clear()
@@ -901,3 +901,16 @@ def test_pull_stats(tmp_dir, dvc, local_remote, caplog):
     with caplog.at_level(level=logging.INFO, logger="dvc"):
         main(["pull"])
     assert "Everything is up to date." in caplog.text
+
+
+def test_local_remote_should_retain_cache_mode(tmp_dir, dvc):
+    tmp_dir.setup_remote()
+    remote = dvc.cloud.get_remote("upstream")
+
+    (stage,) = tmp_dir.dvc_gen("file", "file content")
+    out = first(stage.outs)
+
+    dvc.push()
+
+    remote_path = remote.checksum_to_path(out.checksum)
+    assert os.stat(remote_path).st_mode == os.stat(out.cache_path).st_mode

tests/func/test_external_repo.py

@@ -44,7 +44,7 @@ def test_source_change(erepo_dir):
 
 
 def test_cache_reused(erepo_dir, mocker):
-    with erepo_dir.chdir():
+    with erepo_dir.local_remote_context():
         erepo_dir.dvc_gen("file", "text", commit="add file")
 
     download_spy = mocker.spy(LocalRemote, "download")
@@ -63,6 +63,8 @@ def test_cache_reused(erepo_dir, mocker):
 
 
 def test_known_sha(erepo_dir):
+    erepo_dir.scm.commit("init")
+
     url = "file://{}".format(erepo_dir)
     with external_repo(url) as repo:
         rev = repo.scm.get_rev()
@@ -100,9 +102,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
 
     upstream_dir = tmp_dir
     upstream_url = relpath(upstream_dir, erepo_dir)
-    with erepo_dir.dvc.config.edit() as conf:
-        conf["remote"]["upstream"] = {"url": upstream_url}
-        conf["core"]["remote"] = "upstream"
+    erepo_dir.setup_remote(upstream_url)
 
     erepo_dir.scm_add(
         erepo_dir.dvc.config.files["repo"], commit="Update dvc config"

tests/func/test_gc.py

@@ -240,7 +240,7 @@ def test_gc_without_workspace_raises_error(tmp_dir, dvc):
 
 def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir):
     dvc = erepo_dir.dvc
-    with erepo_dir.chdir():
+    with erepo_dir.local_remote_context():
         from dvc.exceptions import InvalidArgumentError
 
         with pytest.raises(InvalidArgumentError):
@@ -296,12 +296,7 @@ def test_gc_with_possible_args_positive(tmp_dir, dvc):
 
 
 def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory):
-    with dvc.config.edit() as conf:
-        storage = fspath(tmp_path_factory.mktemp("test_remote_base"))
-        conf["remote"]["local_remote"] = {"url": storage}
-        conf["core"]["remote"] = "local_remote"
-
-    dvc.push()
+    tmp_dir.setup_remote()
 
     for flag in ["-cw", "-ca", "-cT", "-caT", "-cwT"]:
         assert main(["gc", "-vf", flag]) == 0

tests/func/test_get.py

@@ -198,7 +198,7 @@ def test_get_file_from_dir(tmp_dir, erepo_dir):
 
 
 def test_get_url_positive(tmp_dir, erepo_dir, caplog):
-    with erepo_dir.chdir():
+    with erepo_dir.local_remote_context():
         erepo_dir.dvc_gen("foo", "foo")
 
     caplog.clear()
@@ -224,16 +224,14 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):
         assert "failed to show URL" in caplog.text
 
 
-def test_get_pipeline_tracked_outs(
-    tmp_dir, dvc, scm, git_dir, local_remote, run_copy
-):
+def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy):
     from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
 
-    tmp_dir.gen("foo", "foo")
-    run_copy("foo", "bar", name="copy-foo-bar")
+    with tmp_dir.local_remote_context():
+        tmp_dir.gen("foo", "foo")
+        run_copy("foo", "bar", name="copy-foo-bar")
     dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
     dvc.scm.commit("add pipeline stage")
-    dvc.push()
 
     with git_dir.chdir():
         Repo.get("file:///{}".format(fspath(tmp_dir)), "bar", out="baz")