diff --git a/tests/remotes.py b/tests/remotes.py index a80619b174..e2a6ed6790 100644 --- a/tests/remotes.py +++ b/tests/remotes.py @@ -78,9 +78,9 @@ class S3Mocked(S3): @classmethod @contextmanager - def remote(cls): + def remote(cls, repo): with mock_s3(): - yield RemoteS3(None, {"url": cls.get_url()}) + yield RemoteS3(repo, {"url": cls.get_url()}) @staticmethod def put_objects(remote, objects): @@ -127,8 +127,8 @@ def get_url(): @classmethod @contextmanager - def remote(cls): - yield RemoteGS(None, {"url": cls.get_url()}) + def remote(cls, repo): + yield RemoteGS(repo, {"url": cls.get_url()}) @staticmethod def put_objects(remote, objects): diff --git a/tests/unit/remote/ssh/test_ssh.py b/tests/unit/remote/ssh/test_ssh.py index ce25665eb4..ac984af0fe 100644 --- a/tests/unit/remote/ssh/test_ssh.py +++ b/tests/unit/remote/ssh/test_ssh.py @@ -1,7 +1,6 @@ import getpass import os import sys -from unittest import TestCase import pytest from mock import mock_open @@ -12,31 +11,31 @@ from tests.remotes import SSHMocked -class TestRemoteSSH(TestCase): - def test_url(self): - user = "test" - host = "123.45.67.89" - port = 1234 - path = "/path/to/dir" +def test_url(dvc): + user = "test" + host = "123.45.67.89" + port = 1234 + path = "/path/to/dir" - # URL ssh://[user@]host.xz[:port]/path - url = "ssh://{}@{}:{}{}".format(user, host, port, path) - config = {"url": url} + # URL ssh://[user@]host.xz[:port]/path + url = "ssh://{}@{}:{}{}".format(user, host, port, path) + config = {"url": url} - remote = RemoteSSH(None, config) - self.assertEqual(remote.path_info, url) + remote = RemoteSSH(dvc, config) + assert remote.path_info == url - # SCP-like URL ssh://[user@]host.xz:/absolute/path - url = "ssh://{}@{}:{}".format(user, host, path) - config = {"url": url} + # SCP-like URL ssh://[user@]host.xz:/absolute/path + url = "ssh://{}@{}:{}".format(user, host, path) + config = {"url": url} - remote = RemoteSSH(None, config) - self.assertEqual(remote.path_info, url) + remote = RemoteSSH(dvc, config) + assert remote.path_info == url - def test_no_path(self): - config = {"url": "ssh://127.0.0.1"} - remote = RemoteSSH(None, config) - self.assertEqual(remote.path_info.path, "") + +def test_no_path(dvc): + config = {"url": "ssh://127.0.0.1"} + remote = RemoteSSH(dvc, config) + assert remote.path_info.path == "" mock_ssh_config = """ @@ -67,9 +66,9 @@ def test_no_path(self): read_data=mock_ssh_config, ) def test_ssh_host_override_from_config( - mock_file, mock_exists, config, expected_host + mock_file, mock_exists, dvc, config, expected_host ): - remote = RemoteSSH(None, config) + remote = RemoteSSH(dvc, config) mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) @@ -96,8 +95,8 @@ def test_ssh_host_override_from_config( new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_user(mock_file, mock_exists, config, expected_user): - remote = RemoteSSH(None, config) +def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): + remote = RemoteSSH(dvc, config) mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) @@ -121,8 +120,8 @@ def test_ssh_user(mock_file, mock_exists, config, expected_user): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_port(mock_file, mock_exists, config, expected_port): - remote = RemoteSSH(None, config) +def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): + remote = RemoteSSH(dvc, config) mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) @@ -156,8 +155,8 @@ def test_ssh_port(mock_file, mock_exists, config, expected_port): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile): - remote = RemoteSSH(None, config) +def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): + remote = RemoteSSH(dvc, config) mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) @@ -178,15 +177,15 @@ def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_gss_auth(mock_file, mock_exists, config, expected_gss_auth): - remote = RemoteSSH(None, config) +def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): + remote = RemoteSSH(dvc, config) mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) assert remote.gss_auth == expected_gss_auth -def test_hardlink_optimization(tmp_dir, ssh_server): +def test_hardlink_optimization(dvc, tmp_dir, ssh_server): port = ssh_server.test_creds["port"] user = ssh_server.test_creds["username"] @@ -196,7 +195,7 @@ def test_hardlink_optimization(tmp_dir, ssh_server): "user": user, "keyfile": ssh_server.test_creds["key_filename"], } - remote = RemoteSSH(None, config) + remote = RemoteSSH(dvc, config) from_info = remote.path_info / "empty" to_info = remote.path_info / "link" diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index e50f59cf5a..41919e9396 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -1,33 +1,31 @@ -from unittest import TestCase - from dvc.remote.azure import RemoteAZURE -class TestRemoteAZURE(TestCase): - container_name = "container-name" - connection_string = ( - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" - "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" - "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" - "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" +container_name = "container-name" +connection_string = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" +) + + +def test_init_compat(dvc): + url = ( + "azure://ContainerName={container_name};{connection_string}" + ).format( + container_name=container_name, connection_string=connection_string, ) + config = {"url": url} + remote = RemoteAZURE(dvc, config) + assert remote.path_info == "azure://" + container_name + assert remote.connection_string == connection_string - def test_init_compat(self): - url = ( - "azure://ContainerName={container_name};{connection_string}" - ).format( - container_name=self.container_name, - connection_string=self.connection_string, - ) - config = {"url": url} - remote = RemoteAZURE(None, config) - self.assertEqual(remote.path_info, "azure://" + self.container_name) - self.assertEqual(remote.connection_string, self.connection_string) - def test_init(self): - prefix = "some/prefix" - url = "azure://{}/{}".format(self.container_name, prefix) - config = {"url": url, "connection_string": self.connection_string} - remote = RemoteAZURE(None, config) - self.assertEqual(remote.path_info, url) - self.assertEqual(remote.connection_string, self.connection_string) +def test_init(dvc): + prefix = "some/prefix" + url = "azure://{}/{}".format(container_name, prefix) + config = {"url": url, "connection_string": connection_string} + remote = RemoteAZURE(dvc, config) + assert remote.path_info == url + assert remote.connection_string == connection_string diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index d5cad28705..e8cafa3b26 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -1,7 +1,6 @@ -from unittest import TestCase - import math import mock +import pytest from dvc.path_info import PathInfo from dvc.remote.base import RemoteBASE @@ -17,42 +16,36 @@ def __eq__(self, other): CallableOrNone = _CallableOrNone() +REMOTE_CLS = RemoteBASE -class TestRemoteBASE(object): - REMOTE_CLS = RemoteBASE - - -class TestMissingDeps(TestCase, TestRemoteBASE): - def test(self): - requires = {"missing": "missing"} - with mock.patch.object(self.REMOTE_CLS, "REQUIRES", requires): - with self.assertRaises(RemoteMissingDepsError): - self.REMOTE_CLS(None, {}) +def test_missing_deps(dvc): + requires = {"missing": "missing"} + with mock.patch.object(REMOTE_CLS, "REQUIRES", requires): + with pytest.raises(RemoteMissingDepsError): + REMOTE_CLS(dvc, {}) -class TestCmdError(TestCase, TestRemoteBASE): - def test(self): - repo = None - config = {} +def test_cmd_error(dvc): + config = {} - cmd = "sed 'hello'" - ret = "1" - err = "sed: expression #1, char 2: extra characters after command" + cmd = "sed 'hello'" + ret = "1" + err = "sed: expression #1, char 2: extra characters after command" - with mock.patch.object( - self.REMOTE_CLS, - "remove", - side_effect=RemoteCmdError("base", cmd, ret, err), - ): - with self.assertRaises(RemoteCmdError): - self.REMOTE_CLS(repo, config).remove("file") + with mock.patch.object( + REMOTE_CLS, + "remove", + side_effect=RemoteCmdError("base", cmd, ret, err), + ): + with pytest.raises(RemoteCmdError): + REMOTE_CLS(dvc, config).remove("file") @mock.patch.object(RemoteBASE, "_cache_checksums_traverse") @mock.patch.object(RemoteBASE, "_cache_object_exists") -def test_cache_exists(object_exists, traverse): - remote = RemoteBASE(None, {}) +def test_cache_exists(object_exists, traverse, dvc): + remote = RemoteBASE(dvc, {}) # remote does not support traverse remote.CAN_TRAVERSE = False @@ -110,8 +103,8 @@ def test_cache_exists(object_exists, traverse): @mock.patch.object( RemoteBASE, "path_to_checksum", side_effect=lambda x: x, ) -def test_cache_checksums_traverse(path_to_checksum, cache_checksums): - remote = RemoteBASE(None, {}) +def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc): + remote = RemoteBASE(dvc, {}) remote.path_info = PathInfo("foo") # parallel traverse @@ -135,8 +128,8 @@ def test_cache_checksums_traverse(path_to_checksum, cache_checksums): ) -def test_cache_checksums(): - remote = RemoteBASE(None, {}) +def test_cache_checksums(dvc): + remote = RemoteBASE(dvc, {}) remote.path_info = PathInfo("foo") with mock.patch.object( diff --git a/tests/unit/remote/test_gdrive.py b/tests/unit/remote/test_gdrive.py index 2e9c8f7917..50aac91903 100644 --- a/tests/unit/remote/test_gdrive.py +++ b/tests/unit/remote/test_gdrive.py @@ -1,7 +1,6 @@ import pytest import os -from dvc.config import Config from dvc.remote.gdrive import ( RemoteGDrive, GDriveAccessTokenRefreshError, @@ -13,11 +12,6 @@ USER_CREDS_MISSED_KEY_ERROR = "{}" -class Repo(object): - tmp_dir = "" - config = Config() - - class TestRemoteGDrive(object): CONFIG = { "url": "gdrive://root/data", @@ -25,12 +19,12 @@ class TestRemoteGDrive(object): "gdrive_client_secret": "secret", } - def test_init(self): - remote = RemoteGDrive(Repo(), self.CONFIG) + def test_init(self, dvc): + remote = RemoteGDrive(dvc, self.CONFIG) assert str(remote.path_info) == self.CONFIG["url"] - def test_drive(self): - remote = RemoteGDrive(Repo(), self.CONFIG) + def test_drive(self, dvc): + remote = RemoteGDrive(dvc, self.CONFIG) os.environ[ RemoteGDrive.GDRIVE_CREDENTIALS_DATA ] = USER_CREDS_TOKEN_REFRESH_ERROR @@ -38,7 +32,7 @@ def test_drive(self): remote._drive os.environ[RemoteGDrive.GDRIVE_CREDENTIALS_DATA] = "" - remote = RemoteGDrive(Repo(), self.CONFIG) + remote = RemoteGDrive(dvc, self.CONFIG) os.environ[ RemoteGDrive.GDRIVE_CREDENTIALS_DATA ] = USER_CREDS_MISSED_KEY_ERROR diff --git a/tests/unit/remote/test_gs.py b/tests/unit/remote/test_gs.py index 62923e41e2..1485d0f1a7 100644 --- a/tests/unit/remote/test_gs.py +++ b/tests/unit/remote/test_gs.py @@ -1,5 +1,3 @@ -from unittest import TestCase - import mock import pytest import requests @@ -8,38 +6,40 @@ from dvc.remote.gs import RemoteGS -class TestRemoteGS(TestCase): - BUCKET = "bucket" - PREFIX = "prefix" - URL = "gs://{}/{}".format(BUCKET, PREFIX) - CREDENTIALPATH = "/path/to/gcp_credentials.json" - PROJECT = "PROJECT" - CONFIG = { - "projectname": PROJECT, - "url": URL, - "credentialpath": CREDENTIALPATH, - } - - def test_init(self): - remote = RemoteGS(None, self.CONFIG) - self.assertEqual(remote.path_info, self.URL) - self.assertEqual(remote.projectname, self.PROJECT) - self.assertEqual(remote.credentialpath, self.CREDENTIALPATH) - - @mock.patch("google.cloud.storage.Client.from_service_account_json") - def test_gs(self, mock_client): - remote = RemoteGS(None, self.CONFIG) - self.assertTrue(remote.credentialpath) - remote.gs() - mock_client.assert_called_once_with(self.CREDENTIALPATH) - - @mock.patch("google.cloud.storage.Client") - def test_gs_no_credspath(self, mock_client): - config = self.CONFIG.copy() - del config["credentialpath"] - remote = RemoteGS(None, config) - remote.gs() - mock_client.assert_called_with(self.PROJECT) +BUCKET = "bucket" +PREFIX = "prefix" +URL = "gs://{}/{}".format(BUCKET, PREFIX) +CREDENTIALPATH = "/path/to/gcp_credentials.json" +PROJECT = "PROJECT" +CONFIG = { + "projectname": PROJECT, + "url": URL, + "credentialpath": CREDENTIALPATH, +} + + +def test_init(dvc): + remote = RemoteGS(dvc, CONFIG) + assert remote.path_info == URL + assert remote.projectname == PROJECT + assert remote.credentialpath == CREDENTIALPATH + + +@mock.patch("google.cloud.storage.Client.from_service_account_json") +def test_gs(mock_client, dvc): + remote = RemoteGS(dvc, CONFIG) + assert remote.credentialpath + remote.gs() + mock_client.assert_called_once_with(CREDENTIALPATH) + + +@mock.patch("google.cloud.storage.Client") +def test_gs_no_credspath(mock_client, dvc): + config = CONFIG.copy() + del config["credentialpath"] + remote = RemoteGS(dvc, config) + remote.gs() + mock_client.assert_called_with(PROJECT) def test_dynamic_chunk_size(): diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index 5831eebe4a..af1cb782d8 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -1,25 +1,23 @@ -from unittest import TestCase - from dvc.remote.oss import RemoteOSS -class TestRemoteOSS(TestCase): - bucket_name = "bucket-name" - endpoint = "endpoint" - key_id = "Fq2UVErCz4I6tq" - key_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" +bucket_name = "bucket-name" +endpoint = "endpoint" +key_id = "Fq2UVErCz4I6tq" +key_secret = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + - def test_init(self): - prefix = "some/prefix" - url = "oss://{}/{}".format(self.bucket_name, prefix) - config = { - "url": url, - "oss_key_id": self.key_id, - "oss_key_secret": self.key_secret, - "oss_endpoint": self.endpoint, - } - remote = RemoteOSS(None, config) - self.assertEqual(remote.path_info, url) - self.assertEqual(remote.endpoint, self.endpoint) - self.assertEqual(remote.key_id, self.key_id) - self.assertEqual(remote.key_secret, self.key_secret) +def test_init(dvc): + prefix = "some/prefix" + url = "oss://{}/{}".format(bucket_name, prefix) + config = { + "url": url, + "oss_key_id": key_id, + "oss_key_secret": key_secret, + "oss_endpoint": endpoint, + } + remote = RemoteOSS(dvc, config) + assert remote.path_info == url + assert remote.endpoint == endpoint + assert remote.key_id == key_id + assert remote.key_secret == key_secret diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index 557ab87f48..59e91713ab 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -30,9 +30,9 @@ def test_remote_without_checksum_jobs_default(dvc): @pytest.mark.parametrize("remote_cls", [RemoteGS, RemoteS3]) -def test_makedirs_not_create_for_top_level_path(remote_cls, mocker): +def test_makedirs_not_create_for_top_level_path(remote_cls, dvc, mocker): url = "{.scheme}://bucket/".format(remote_cls) - remote = remote_cls(None, {"url": url}) + remote = remote_cls(dvc, {"url": url}) mocked_client = mocker.PropertyMock() # we use remote clients with same name as scheme to interact with remote mocker.patch.object(remote_cls, remote.scheme, mocked_client) diff --git a/tests/unit/remote/test_remote_dir.py b/tests/unit/remote/test_remote_dir.py index a39ef6981d..342a5af557 100644 --- a/tests/unit/remote/test_remote_dir.py +++ b/tests/unit/remote/test_remote_dir.py @@ -25,10 +25,10 @@ @pytest.fixture -def remote(request): +def remote(request, dvc): if not request.param.should_test(): raise pytest.skip() - with request.param.remote() as remote: + with request.param.remote(dvc) as remote: request.param.put_objects(remote, FILE_WITH_CONTENTS) yield remote @@ -86,11 +86,11 @@ def test_walk_files(remote): @pytest.mark.parametrize("remote", [S3Mocked], indirect=True) -def test_copy_preserve_etag_across_buckets(remote): +def test_copy_preserve_etag_across_buckets(remote, dvc): s3 = remote.s3 s3.create_bucket(Bucket="another") - another = RemoteS3(None, {"url": "s3://another", "region": "us-east-1"}) + another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"}) from_info = remote.path_info / "foo" to_info = another.path_info / "foo" diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py index 57fcfcec79..5cf2f9cab7 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/unit/remote/test_s3.py @@ -19,14 +19,14 @@ def grants(): } -def test_init(): +def test_init(dvc): config = {"url": url} - remote = RemoteS3(None, config) + remote = RemoteS3(dvc, config) assert remote.path_info == url -def test_grants(): +def test_grants(dvc): config = { "url": url, "grant_read": "id=read-permission-id,id=other-read-permission-id", @@ -34,7 +34,7 @@ def test_grants(): "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } - remote = RemoteS3(None, config) + remote = RemoteS3(dvc, config) assert ( remote.extra_args["GrantRead"] @@ -48,9 +48,9 @@ def test_grants(): ) -def test_grants_mutually_exclusive_acl_error(grants): +def test_grants_mutually_exclusive_acl_error(dvc, grants): for grant_option, grant_value in grants.items(): config = {"url": url, "acl": "public-read", grant_option: grant_value} with pytest.raises(ConfigError): - RemoteS3(None, config) + RemoteS3(dvc, config)