Added basic tests

CHANGELOG.md

@@ -1,5 +1,5 @@
 # Change Log
 
-## Version 0.0.1 (2019-10-01)
+## Version 0.0.1 (2020-02-14)
 ### Added
 - Initial release.

Pipfile

@@ -6,6 +6,15 @@ verify_ssl = true
 [dev-packages]
 pytest = "*"
 twine = "*"
+pytest-asyncio = "*"
+pytest-cov = "*"
+coverage = "*"
+coveralls = "*"
+pytest-mock = "*"
+pytest-pylint = "*"
+pylint = "*"
+wheel = "*"
+autopep8 = "*"
 
 [packages]
 synapseclient = "*"

README.md

@@ -1,3 +1,70 @@
 # Synapse Downloader
 
 Utility for downloading large datasets from Synapse.
+
+## Dependencies
+
+- [Python3.7](https://www.python.org/)
+- A [Synapse](https://www.synapse.org/) account with a username/password. Authentication through a 3rd party (.e.g., Google) will not work, you must have a Synapse user/pass for the [API to authenticate](http://docs.synapse.org/python/#connecting-to-synapse).
+
+## Install
+
+```bash
+pip install synapse-downloader
+```
+
+## Configuration
+
+Your Synapse credential can be provided on the command line (`--username`, `--password`) or via environment variables.
+
+```bash
+SYNAPSE_USERNAME=your-synapse-username
+SYNAPSE_PASSWORD=your-synapse-password
+```
+
+## Usage
+
+```text
+usage: synapse-downloader [-h] [-u USERNAME] [-p PASSWORD] [-ll LOG_LEVEL]
+                          [-ld LOG_DIR] [-dt DOWNLOAD_TIMEOUT] [-w] [-wc] [-c]
+                          [-ci [COMPARE_IGNORE]]
+                          entity-id download-path
+
+positional arguments:
+  entity-id             The ID of the Synapse entity to download or compare
+                        (Project, Folder or File).
+  download-path         The local path to save the files to or to compare.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -u USERNAME, --username USERNAME
+                        Synapse username.
+  -p PASSWORD, --password PASSWORD
+                        Synapse password.
+  -ll LOG_LEVEL, --log-level LOG_LEVEL
+                        Set the logging level.
+  -ld LOG_DIR, --log-dir LOG_DIR
+                        Set the directory where the log file will be written.
+  -dt DOWNLOAD_TIMEOUT, --download-timeout DOWNLOAD_TIMEOUT
+                        Set the maximum time (in seconds) a file can download
+                        before it is canceled.
+  -w, --with-view       Use an entity view for loading file info. Fastest for
+                        large projects.
+  -wc, --with-compare   Run the comparison after downloading everything.
+  -c, --compare         Compare a local directory against a remote project or
+                        folder.
+  -ci [COMPARE_IGNORE], --compare-ignore [COMPARE_IGNORE]
+                        Path to directories or files to ignore when comparing.
+
+```
+
+## Development Setup
+
+```bash
+pipenv --three
+pipenv shell
+make pip_install
+make build
+make install_local
+```
+See [Makefile](Makefile) for all commands.

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="synapse-downloader",
-    version="0.0.b39",
+    version="0.0.1",
     author="Patrick Stout",
     author_email="pstout@prevagroup.com",
     license="Apache2",

src/synapse_downloader/cli.py

@@ -55,7 +55,7 @@ def main(args=None):
                         default=SynapseProxy.Aio.FILE_DOWNLOAD_TIMEOUT)
 
     parser.add_argument('-w', '--with-view',
-                        help='Use an entity view for loading file info. Fastest for large projects. Only available for "-s new or basic" and "-c"',
+                        help='Use an entity view for loading file info. Fastest for large projects.',
                         default=False,
                         action='store_true')
 

tests/conftest.py

@@ -0,0 +1,91 @@
+import pytest
+import os
+import json
+import shutil
+import tempfile
+from .synapse_test_helper import SynapseTestHelper
+from src.synapse_downloader.core import SynapseProxy
+
+# Load Environment variables.
+module_dir = os.path.dirname(os.path.abspath(__file__))
+
+test_env_file = os.path.join(module_dir, 'private.test.env.json')
+
+if os.path.isfile(test_env_file):
+    with open(test_env_file) as f:
+        config = json.load(f).get('test')
+
+        for key, value in config.items():
+            os.environ[key] = value
+else:
+    print('WARNING: Test environment file not found at: {0}'.format(test_env_file))
+
+
+@pytest.fixture(scope='module')
+def syn_client():
+    return SynapseProxy.client()
+
+
+@pytest.fixture(scope='module')
+def syn_test_helper():
+    """
+    Provides the SynapseTestHelper as a fixture per session.
+    """
+    helper = SynapseTestHelper()
+    yield helper
+    helper.dispose()
+
+
+@pytest.fixture(scope='module')
+def syn_test_helper_class():
+    """
+    Provides the SynapseTestHelper as a fixture per session.
+    """
+    helper = SynapseTestHelper()
+    yield helper
+    helper.dispose()
+
+
+@pytest.fixture(scope='module')
+def mk_tempdir():
+    created = []
+
+    def _mk():
+        path = tempfile.mkdtemp()
+        created.append(path)
+        return path
+
+    yield _mk
+
+    for path in created:
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+
+
+@pytest.fixture(scope='module')
+def mk_tempfile(mk_tempdir, syn_test_helper):
+    temp_dir = mk_tempdir()
+
+    def _mk(content=syn_test_helper.uniq_name()):
+        fd, tmp_filename = tempfile.mkstemp(dir=temp_dir)
+        with os.fdopen(fd, 'w') as tmp:
+            tmp.write(content)
+        return tmp_filename
+
+    yield _mk
+
+    if os.path.isdir(temp_dir):
+        shutil.rmtree(temp_dir)
+
+
+@pytest.fixture(scope='module')
+def write_file(syn_test_helper):
+    def _write(full_path, content=syn_test_helper.uniq_name()):
+        # Create the directory if it doesn't exist.
+        if not os.path.exists(os.path.dirname(full_path)):
+            os.makedirs(os.path.dirname(full_path))
+
+        with open(full_path, mode='w') as f:
+            f.write(content)
+
+    yield _write

tests/synapse_downloader/compare/test_compare.py

@@ -0,0 +1,61 @@
+import pytest
+import os
+from src.synapse_downloader.compare.comparer import Comparer
+
+
+@pytest.fixture(scope='module')
+def syn_data(syn_test_helper_class, mk_tempdir, write_file):
+    source_dir = mk_tempdir()
+
+    project = syn_test_helper_class.create_project(prefix='Project-')
+    file0 = os.path.join(source_dir, 'file0.txt')
+    write_file(file0)
+    syn_file0 = syn_test_helper_class.create_file(name=os.path.basename(file0), path=file0, parent=project)
+
+    folder1 = os.path.join(source_dir, 'folder1')
+    syn_folder1 = syn_test_helper_class.create_folder(name=os.path.basename(folder1), parent=project)
+
+    file1 = os.path.join(folder1, 'file1.txt')
+    write_file(file1)
+    syn_file1 = syn_test_helper_class.create_file(name=os.path.basename(file1), path=file1, parent=syn_folder1)
+
+    folder2 = os.path.join(folder1, 'folder2')
+    syn_folder2 = syn_test_helper_class.create_folder(name=os.path.basename(folder2), parent=syn_folder1)
+
+    file2 = os.path.join(folder2, 'file2.txt')
+    write_file(file2)
+    syn_file2 = syn_test_helper_class.create_file(name=os.path.basename(file2), path=file2, parent=syn_folder2)
+
+    return {
+        'source_dir': source_dir,
+        'project': project,
+        'file0': file0,
+        'syn_file0': syn_file0,
+        'syn_folder1': syn_folder1,
+        'file1': file1,
+        'syn_file1': syn_file1,
+        'syn_folder2': syn_folder2,
+        'file2': file2,
+        'syn_file2': syn_file2
+    }
+
+
+def test_it_compares_everything(syn_data):
+    source_dir = syn_data['source_dir']
+    project = syn_data['project']
+
+    comparer = Comparer(project.id, source_dir)
+    comparer.start()
+    assert comparer.has_errors is False
+
+
+def test_it_compares_everything_with_entity_view(syn_data, capsys):
+    source_dir = syn_data['source_dir']
+    project = syn_data['project']
+
+    comparer = Comparer(project.id, source_dir)
+    comparer.start()
+    assert comparer.has_errors is False
+    # TODO: Figure out how to test the view was used.
+
+# TODO: Add additional tests...

tests/synapse_downloader/download/test_downloader.py

@@ -0,0 +1,79 @@
+import pytest
+import os
+from src.synapse_downloader.download.downloader import Downloader
+
+
+@pytest.fixture(scope='module')
+def syn_data(syn_test_helper_class, mk_tempdir, mk_tempfile):
+    download_dir = mk_tempdir()
+
+    project = syn_test_helper_class.create_project(prefix='Project-')
+    file0 = mk_tempfile()
+    syn_file0 = syn_test_helper_class.create_file(prefix='file0-', path=file0, parent=project)
+
+    syn_folder1 = syn_test_helper_class.create_folder(prefix='folder1-', parent=project)
+    file1 = mk_tempfile()
+    syn_file1 = syn_test_helper_class.create_file(prefix='file1-', path=file1, parent=syn_folder1)
+
+    syn_folder2 = syn_test_helper_class.create_folder(prefix='folder2-', parent=syn_folder1)
+    file2 = mk_tempfile()
+    syn_file2 = syn_test_helper_class.create_file(prefix='file2-', path=file2, parent=syn_folder2)
+    return {
+        'download_dir': download_dir,
+        'project': project,
+        'file0': file0,
+        'syn_file0': syn_file0,
+        'syn_folder1': syn_folder1,
+        'file1': file1,
+        'syn_file1': syn_file1,
+        'syn_folder2': syn_folder2,
+        'file2': file2,
+        'syn_file2': syn_file2
+    }
+
+
+def assert_local_download_data(syn_data):
+    download_dir = syn_data['download_dir']
+    file0 = syn_data['file0']
+    syn_folder1 = syn_data['syn_folder1']
+    file1 = syn_data['file1']
+    syn_folder2 = syn_data['syn_folder2']
+    file2 = syn_data['file2']
+
+    file0_local_path = os.path.join(download_dir, os.path.basename(file0))
+    assert os.path.isfile(file0_local_path)
+
+    folder1_local_path = os.path.join(download_dir, syn_folder1.name)
+    assert os.path.isdir(folder1_local_path)
+
+    file1_local_path = os.path.join(folder1_local_path, os.path.basename(file1))
+    assert os.path.isfile(file1_local_path)
+
+    folder2_local_path = os.path.join(folder1_local_path, syn_folder2.name)
+    assert os.path.isdir(folder2_local_path)
+
+    file2_local_path = os.path.join(folder2_local_path, os.path.basename(file2))
+    assert os.path.isfile(file2_local_path)
+
+
+def test_it_downloads_everything(syn_data):
+    download_dir = syn_data['download_dir']
+    project = syn_data['project']
+
+    downloader = Downloader(project.id, download_dir)
+    downloader.start()
+    assert downloader.has_errors is False
+    assert_local_download_data(syn_data)
+
+
+def test_it_downloads_everything_with_entity_view(syn_data):
+    download_dir = syn_data['download_dir']
+    project = syn_data['project']
+
+    downloader = Downloader(project.id, download_dir, with_view=True)
+    downloader.start()
+    assert downloader.has_errors is False
+    # TODO: Figure out how to test the view was used.
+    assert_local_download_data(syn_data)
+
+# TODO: Add additional tests...

tests/synapse_downloader/test_cli.py

@@ -3,4 +3,5 @@
 
 
 def test_cli():
-    main(['syn21064576', '/tmp/syn-download', '-c'])
+    # TODO: test this
+    pass