add more unit tests on downloader and argpare

dcard/cli.py

@@ -31,26 +31,26 @@ def main(args=None):
     if args.verbose:
         dcard.add_handles_on_logger()
     if args.mode == 'download':
-        if not (args.forum or args.pages):
-            parser.error('No action requested, add --forum or --pages')
+        if not (args.forum or args.number):
+            parser.error('No action requested, add --forum or --number')
         download(args)
 
 
 def download(args):
 
-    def collect_ids(metas):
-        return [meta['id'] for meta in metas if meta['likeCount'] >= likes_thesh]
+    def metas_filter(metas):
+        return [meta for meta in metas if meta['likeCount'] >= likes_thesh]
 
     likes_thesh = args.likes_threshold if args.likes_threshold else 0
 
     dcard = Dcard()
 
     start_time = time.time()
 
-    ids = dcard \
+    metas = dcard \
         .forums(args.forum) \
-        .get_metas(num=args.number, callback=collect_ids)
-    posts = dcard.posts(ids).get(comments=False, links=False)
+        .get_metas(num=args.number, callback=metas_filter)
+    posts = dcard.posts(metas).get(comments=False, links=False)
 
     if args.flatten:
         posts.downloader.subfolder_pattern = '[{likeCount}推] {id}-{folder_name}'
@@ -59,7 +59,7 @@ def collect_ids(metas):
         posts.downloader.resources_folder = args.output
 
     resources = posts.parse_resources()
-    status = posts.download(resources)
+    status, fails = posts.download(resources)
 
     print('成功下載 %d items！' % posts.downloader.done_resources
           if all(status) else '出了點錯下載不完全喔')

dcard/manager.py

@@ -26,15 +26,13 @@
 def download(task):
     filepath, src = task
     if os.path.exists(filepath):
-        return True
+        return True, src
     response = client.get_stream(src)
     if response.ok:
         with open(filepath, 'wb') as stream:
             for chunk in response.iter_content(chunk_size=1024):
                 stream.write(chunk)
-    else:
-        print('%s can not download.' % src)
-    return response.ok
+    return (response.ok, src)
 
 
 class Downloader:
@@ -60,12 +58,14 @@ def download(self):
             self.done_resources += len(urls)
             tasks += [(self._gen_filepath(meta, url), url) for url in urls]
 
-        # results = list(new_download(tasks))
         with contextlib.closing(Pool(8)) as pool:
             async_results = pool.map_async(download, tasks)
             results = async_results.get()
-        logger.info('[Downloader] finish {0} items!'.format(len(results)))
-        return results
+
+        status = [ok for ok, _ in results]
+        fails = [src for ok, src in results if not ok]
+        logger.info('[Downloader] finish {0} items!'.format(len(status)))
+        return status, fails
 
     def _gen_filepath(self, meta, url):
         folder = self._gen_full_folder(meta)

tests/test_cli.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from dcard.cli import parser, main, download
 
 
@@ -12,6 +14,13 @@ def test_verbose_log(self):
         main(args)
         assert os.path.exists('dcard.log')
 
+    def test_download_but_no_params(self):
+        argv = 'download'.split()
+        args = parser.parse_args(argv)
+
+        with pytest.raises(SystemExit):
+            main(args)
+
     def test_download_basic(self):
         argv = 'download -f funny -n 15'.split()
         args = parser.parse_args(argv)

tests/test_manager.py

@@ -0,0 +1,15 @@
+from dcard.manager import Downloader
+
+
+class TestDownloader:
+
+    def test_dwonload_with_bundles_but_no_urls(self):
+        downloader = Downloader()
+        metas = dict(test='some data')
+        urls = []
+        bundles = [(metas, urls)]
+
+        downloader.set_bundles(bundles)
+        downloader.download()
+
+        assert downloader.done_resources == 0

tests/test_posts.py

@@ -86,5 +86,6 @@ def test_parse_resourses_in_content_and_comments(self, dcard):
     def test_download_resourses(self, dcard):
         posts = dcard.posts(9487).get(comments=False, links=False)
         resources = posts.parse_resources()
-        status = posts.download(resources)
+        status, fails = posts.download(resources)
         assert all(status)
+        assert len(fails) == 0