Add integration tests for Azure Remote

.appveyor.yml

@@ -15,6 +15,8 @@ environment:
     secure: 96fJ3r2i2GohbXHwnSs5N4EplQ7q8YmLpPWM0AC+f4s=
   CODECOV_TOKEN:
     secure: XN4jRtmGE5Bqg8pPZkwNs7kn3UEI73Rkldqc0MGsQISZBm5TNJZOPofDMc1QnUsf
+  AZURE_STORAGE_CONTAINER_NAME: appveyor-tests
+  AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
 
   matrix:
 
@@ -44,6 +46,9 @@ environment:
 #      PYTHON_ARCH: "64"
 
 install:
+  - ps: Install-Product node
+  - npm install -g azurite
+  - ps: $AzuriteProcess = Start-Process azurite-blob -PassThru
   - cinst wget
   - cinst awscli
   - cinst gcloudsdk
@@ -71,6 +76,9 @@ after_test:
   - if "%APPVEYOR_REPO_TAG%"=="true" python -m dvc pull
   - if "%APPVEYOR_REPO_TAG%"=="true" .\scripts\build_windows.cmd
 
+on_finish:
+  - ps: Stop-Process $AzuriteProcess
+
 artifacts:
   - path: dvc*.exe
     name: DVC Installer

dvc/remote/azure.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import os
 import re
 import time
@@ -29,14 +30,14 @@ def __call__(self, current, total):
 class RemoteAzure(RemoteBase):
     scheme = 'azure'
     REGEX = (r'^azure://'
-             r'(ContainerName=(?P<container_name>[^;]+);)?'
+             r'(ContainerName=(?P<container_name>[^;]+);?)?'
              r'(?P<connection_string>.+)?$')
     REQUIRES = {'azure-storage-blob': BlockBlobService}
     PARAM_ETAG = 'etag'
     COPY_POLL_SECONDS = 5
 
     def __init__(self, project, config):
-        super().__init__(project, config)
+        super(RemoteAzure, self).__init__(project, config)
         self.project = project
 
         url = config.get(Config.SECTION_REMOTE_URL)
@@ -137,19 +138,16 @@ def md5s_to_path_infos(self, md5s):
         } for md5 in md5s]
 
     def exists(self, path_infos):
-        # TODO: check whether per-key exists() is faster than listing all
-        # objects altogether.
-        ret = []
+        keys = {blob.name
+                for blob in self.blob_service.list_blobs(self.bucket)}
 
+        ret = []
         for path_info in path_infos:
             if path_info['scheme'] != self.scheme:
                 raise NotImplementedError
 
-            bucket = path_info['bucket']
             key = path_info['key']
-
-            exists = self.blob_service.exists(bucket, key)
-            ret.append(exists)
+            ret.append(key in keys)
 
         return ret
 

scripts/ci/before_install.sh

@@ -17,3 +17,10 @@ ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
 ssh 0.0.0.0 ls &> /dev/null
 ssh 127.0.0.1 ls &> /dev/null
 ssh localhost ls &> /dev/null
+
+scriptdir="$(dirname $0)"
+
+if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ]; then
+  bash "$scriptdir/install_azurite.sh"
+  source ~/.bashrc
+fi

scripts/ci/install_azurite.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# install docker
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce
+
+# run azurite
+sudo docker run --restart always -e executable=blob -p 10000:10000 --tmpfs /opt/azurite/folder -d arafato/azurite:2.6.5
+
+# save secrets
+echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> ~/.bashrc
+echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> ~/.bashrc

tests/__main__.py

@@ -7,4 +7,4 @@
 os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), os.getenv('PATH')))
 os.putenv('DVC_HOME', REPO_ROOT)
 
-check_call('nosetests  --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)
+check_call('nosetests -v --processes=-1 --process-timeout=200 --cover-inclusive --cover-erase --cover-package=dvc --with-coverage', shell=True)

tests/test_data_cloud.py

@@ -1,6 +1,8 @@
+from subprocess import CalledProcessError
+from subprocess import check_output
+from unittest import SkipTest
 import os
 import time
-import stat
 import uuid
 import shutil
 import getpass
@@ -9,7 +11,8 @@
 
 from dvc.main import main
 from dvc.config import Config, ConfigError
-from dvc.data_cloud import DataCloud, RemoteS3, RemoteGS, RemoteAzure, RemoteLOCAL, RemoteSSH, RemoteHDFS
+from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure,
+                            RemoteLOCAL, RemoteSSH, RemoteHDFS)
 from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
 
 from tests.basic_env import TestDvc
@@ -46,8 +49,11 @@ def _should_test_gcp():
         if os.path.exists(creds):
             os.unlink(creds)
         shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
-        ret = os.system('gcloud auth activate-service-account --key-file={}'.format(creds))
-        assert ret == 0
+        try:
+            check_output(['gcloud', 'auth', 'activate-service-account',
+                          '--key-file', creds])
+        except CalledProcessError:
+            return False
         return True
 
     return False
@@ -59,24 +65,36 @@ def _should_test_azure():
     elif os.getenv("DVC_TEST_AZURE") == "false":
         return False
 
-    return os.getenv("AZURE_STORAGE_CONTAINER") and os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    return (os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+            and os.getenv("AZURE_STORAGE_CONNECTION_STRING"))
 
 
 def _should_test_ssh():
     if os.getenv("DVC_TEST_SSH") == "true":
         return True
 
-    #FIXME: enable on windows
+    # FIXME: enable on windows
     if os.name == 'nt':
         return False
 
-    assert os.system('ssh 127.0.0.1 ls &> /dev/null') == 0
+    try:
+        check_output(['ssh', '127.0.0.1', 'ls'])
+    except (CalledProcessError, FileNotFoundError):
+        return False
 
     return True
 
 
 def _should_test_hdfs():
-    return platform.system() == 'Linux'
+    if platform.system() != 'Linux':
+        return False
+
+    try:
+        check_output(['hadoop', 'version'])
+    except (CalledProcessError, FileNotFoundError):
+        return False
+
+    return True
 
 
 def get_local_storagepath():
@@ -88,11 +106,15 @@ def get_local_url():
 
 
 def get_ssh_url():
-    return 'ssh://{}@127.0.0.1:{}'.format(getpass.getuser(), get_local_storagepath())
+    return 'ssh://{}@127.0.0.1:{}'.format(
+        getpass.getuser(),
+        get_local_storagepath())
 
 
 def get_hdfs_url():
-    return 'hdfs://{}@127.0.0.1{}'.format(getpass.getuser(), get_local_storagepath())
+    return 'hdfs://{}@127.0.0.1{}'.format(
+        getpass.getuser(),
+        get_local_storagepath())
 
 
 def get_aws_storagepath():
@@ -134,28 +156,39 @@ def test(self):
 
         for scheme, cl in [('s3://', RemoteS3),
                            ('gs://', RemoteGS),
+                           ('azure://ContainerName=', RemoteAzure),
                            ('ssh://user@localhost:/', RemoteSSH),
                            (tempfile.mkdtemp(), RemoteLOCAL)]:
-            config[TEST_SECTION][Config.SECTION_REMOTE_URL] = scheme + str(uuid.uuid4())
+
+            remote_url = scheme + str(uuid.uuid4())
+            config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
             self._test_cloud(config, cl)
 
     def test_unsupported(self):
-        with self.assertRaises(ConfigError) as cx:
+        with self.assertRaises(ConfigError):
+            remote_url = 'notsupportedscheme://a/b'
             config = TEST_CONFIG
-            config[TEST_SECTION][Config.SECTION_REMOTE_URL] = 'notsupportedscheme://a/b'
+            config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url
             DataCloud(self.dvc, config=config)
 
 
 class TestDataCloudBase(TestDvc):
+    def _get_cloud_class(self):
+        return None
+
     def _should_test(self):
         return False
 
     def _get_url(self):
-        return None
+        return ''
 
-    def _setup_cloud(self):
+    def _ensure_should_run(self):
         if not self._should_test():
-            return
+            raise SkipTest('Test {} is disabled'
+                           .format(self.__class__.__name__))
+
+    def _setup_cloud(self):
+        self._ensure_should_run()
 
         repo = self._get_url()
 
@@ -229,8 +262,8 @@ def _test_cloud(self):
         self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
     def test(self):
-        if self._should_test():
-            self._test_cloud()
+        self._ensure_should_run()
+        self._test_cloud()
 
 
 class TestRemoteS3(TestDataCloudBase):
@@ -328,7 +361,7 @@ def _test_cloud(self, remote=None):
         stage_dir = self.dvc.add(self.DATA_DIR)
         cache_dir = stage_dir.outs[0].cache
 
-        #FIXME check status output
+        # FIXME check status output
         sleep()
         self.main(['status', '-c'] + args)