[win] _download fix

hikiko4ern · Mar 25, 2019 · d05eed9 · d05eed9
1 parent 0fe33e5
commit d05eed9
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 30 deletions.
diff --git a/dump.py b/dump.py
@@ -21,7 +21,7 @@
 from youtube_dl import YoutubeDL
 
 NAME = 'VK Dump Tool'
-VERSION = '0.9.1'
+VERSION = '0.9.2'
 API_VERSION = '5.92'
 
 
@@ -355,7 +355,7 @@ def update(self, dmp, **kwargs):
         dmp: Dumper obj
         """
         import itertools
-        from multiprocess import Pool
+        from multiprocessing import Pool
         from modules.utils import copy_func
 
         def apply_args_and_kwargs(fn, args, kwargs):
@@ -571,7 +571,19 @@ def _settings_save(self):
             config['EXCLUDED_DIALOGS'] = {'id': ','.join([str(i) for i in Dumper._EXCLUDED_DIALOGS])}
             config.write(cf)
 
-    def _download(obj, folder, **kwargs):
+    def _download(dmp, obj, folder, **kwargs):
+        """
+        dmp: Dumper class
+        """
+        # import ipdb
+        # ipdb.set_trace()
+        from os import name as osname
+        if osname == 'nt':
+            import os
+            import os.path
+            import requests
+            # import shutil
+
         if not obj:
             return False
 
@@ -583,7 +595,7 @@ def _download(obj, folder, **kwargs):
             kwargs = obj
 
         if 'name' in kwargs:
-            fn = '_'.join(kwargs['name'].split(' ')) if Dumper._settings['REPLACE_SPACES'] else kwargs['name']
+            fn = '_'.join(kwargs['name'].split(' ')) if dmp._settings['REPLACE_SPACES'] else kwargs['name']
             if 'ext' in kwargs:
                 if fn.split('.')[-1] != kwargs['ext']:
                     fn += '.{}'.format(kwargs['ext'])
@@ -596,8 +608,8 @@ def _download(obj, folder, **kwargs):
         if 'access_key' in kwargs:
             url = '{}?access_key={ak}'.format(url, ak=kwargs['access_key'])
 
-        for c in Dumper._INVALID_CHARS:
-            fn = fn.replace(c, Dumper._settings['REPLACE_CHAR'])
+        for c in dmp._INVALID_CHARS:
+            fn = fn.replace(c, dmp._settings['REPLACE_CHAR'])
 
         if not os.path.exists(os.path.join(folder, fn)) or kwargs.get('force'):
             try:
@@ -621,9 +633,17 @@ def _download(obj, folder, **kwargs):
         else:
             return True
 
-    def _download_video(v, folder):
+    def _download_video(dmp, v, folder):
+        """
+        dmp: Dumper class
+        """
+        from os import name as osname
+        if osname == 'nt':
+            from urllib.request import urlopen
+            from re import search as research
+
         if 'platform' in v:
-            return Dumper._download_external(v['player'], folder)
+            return dmp._download_external(v['player'], folder)
         else:
             if 'player' not in v:
                 return False
@@ -635,12 +655,13 @@ def _download_video(v, folder):
             url = v['player'] if ('access_key' not in v) else f"{v['player']}?access_key={v['access_key']}"
             data = urlopen(url).read()
             try:
-                return Dumper._download(research(b'https://cs.*vkuservideo.*'
-                                                 + str(min(v['height'], v['width']) if ('width' in v) else v['height']).encode()
-                                                 + b'.mp4', data).group(0).decode(),
-                                        folder,
-                                        name=v['title'] + '_' + str(v['id']),
-                                        ext='mp4')
+                return dmp._download(dmp,
+                                     research(b'https://cs.*vkuservideo.*'
+                                              + str(min(v['height'], v['width']) if ('width' in v) else v['height']).encode()
+                                              + b'.mp4', data).group(0).decode(),
+                                     folder,
+                                     name=v['title'] + '_' + str(v['id']),
+                                     ext='mp4')
             except AttributeError:
                 return False
 

diff --git a/modules/attachments.py b/modules/attachments.py
@@ -118,7 +118,8 @@ def dump_attachments_only(dmp):
 
             with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                 res = pool.starmap(copy_func(dmp._download),
-                                   zip(map(lambda t: sorted(t['attachment']['photo']['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
+                                   zip(itertools.repeat(dmp.__class__),
+                                       map(lambda t: sorted(t['attachment']['photo']['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
                                        itertools.repeat(af)))
 
             print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
@@ -143,7 +144,9 @@ def dump_attachments_only(dmp):
             try:
                 with Pool(dmp._AVAILABLE_THREADS if dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp._settings['POOL_PROCESSES']) as pool:
                     res = pool.starmap(copy_func(dmp._download_video),
-                                       zip(map(lambda t: t['attachment']['video'], video['items']), itertools.repeat(af)))
+                                       zip(itertools.repeat(dmp.__class__),
+                                           map(lambda t: t['attachment']['video'], video['items']),
+                                           itertools.repeat(af)))
                     print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
                                                                   len(video['items']),
                                                                   len(next(os.walk(af))[2])))
@@ -168,7 +171,9 @@ def dump_attachments_only(dmp):
 
             with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                 res = pool.starmap(copy_func(dmp._download),
-                                   zip(map(lambda t: t['attachment']['doc'], docs['items']), itertools.repeat(af)))
+                                   zip(itertools.repeat(dmp.__class__),
+                                       map(lambda t: t['attachment']['doc'], docs['items']),
+                                       itertools.repeat(af)))
 
             print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
                                                           len(docs['items']),

diff --git a/modules/audio.py b/modules/audio.py
@@ -37,7 +37,7 @@ def dump_audio(dmp):
         print('  .../{}'.format(len(tracks)), end='\r')
         with Pool(dmp._settings['POOL_PROCESSES']) as pool:
             res = pool.starmap(copy_func(dmp._download),
-                               zip(audios, itertools.repeat(folder)))
+                               zip(itertools.repeat(dmp.__class__), audios, itertools.repeat(folder)))
 
         print('\x1b[2K  {}/{} (total: {})'.format(sum(filter(None, res)),
                                                   len(audios),

diff --git a/modules/docs.py b/modules/docs.py
@@ -34,7 +34,7 @@ def dump_docs(dmp):
         print('  .../{}'.format(docs['count']), end='\r')
         with Pool(dmp._settings['POOL_PROCESSES']) as pool:
             res = pool.starmap(copy_func(dmp._download),
-                               zip(objs, itertools.repeat(folder)))
+                               zip(itertools.repeat(dmp.__class__), objs, itertools.repeat(folder)))
 
         print('\x1b[2K    {}/{} (total: {})'.format(sum(filter(None, res)),
                                                     len(objs),

diff --git a/modules/fave.py b/modules/fave.py
@@ -110,22 +110,22 @@ def dump_fave_posts(dmp):
         print('  [фото ({})]'.format(len(photo)))
         with Pool(dmp._settings['POOL_PROCESSES']) as pool:
             pool.starmap(copy_func(dmp._download),
-                         zip(photo, itertools.repeat(folder_photo)))
+                         zip(itertools.repeat(dmp.__class__), photo, itertools.repeat(folder_photo)))
 
     try:
         if video:
             print('  [видео ({})]'.format(len(video['items'])))
             with Pool(dmp._settings['POOL_PROCESSES'] if not dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp._AVAILABLE_THREADS) as pool:
                 pool.starmap(copy_func(dmp._download_video),
-                             zip(video['items'], itertools.repeat(folder_video)))
+                             zip(itertools.repeat(dmp.__class__), video['items'], itertools.repeat(folder_video)))
     except MaybeEncodingError:
         None
 
     if docs:
         print('  [документы ({})]'.format(len(docs)))
         with Pool(dmp._settings['POOL_PROCESSES']) as pool:
             pool.starmap(copy_func(dmp._download),
-                         zip(docs, itertools.repeat(folder_docs)))
+                         zip(itertools.repeat(dmp.__class__), docs, itertools.repeat(folder_docs)))
 
 
 def dump_fave_photo(dmp):
@@ -148,7 +148,8 @@ def dump_fave_photo(dmp):
         print('  .../{}'.format(photo['count']), end='\r')
         with Pool(dmp._settings['POOL_PROCESSES']) as pool:
             res = pool.starmap(copy_func(dmp._download),
-                               zip(map(lambda p: sorted(p['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
+                               zip(itertools.repeat(dmp.__class__),
+                                   map(lambda p: sorted(p['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
                                    itertools.repeat(folder)))
         print('\x1b[2K  {}/{} (total: {})'.format(sum(filter(None, res)),
                                                   photo['count'],
@@ -191,7 +192,10 @@ def dump_fave_video(dmp):
         print('    .../{}'.format(video['count']), end='\r')
         try:
             with Pool(dmp._AVAILABLE_THREADS if dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp._settings['POOL_PROCESSES']) as pool:
-                res = pool.starmap(copy_func(dmp._download_video), zip(video['items'], itertools.repeat(folder)))
+                res = pool.starmap(copy_func(dmp._download_video),
+                                   zip(itertools.repeat(dmp.__class__),
+                                       video['items'],
+                                       itertools.repeat(folder)))
             print('\x1b[2K    {}/{} (total: {})'.format(sum([1 for i in res if i is True]),
                                                         video['count'],
                                                         len(next(os.walk(folder))[2])))

diff --git a/modules/messages.py b/modules/messages.py
@@ -498,7 +498,9 @@ def sortById(msg):
 
             with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                 res = pool.starmap(copy_func(dmp._download),
-                                   zip(attachments['audio_messages'], itertools.repeat(af)))
+                                   zip(itertools.repeat(dmp.__class__),
+                                       attachments['audio_messages'],
+                                       itertools.repeat(af)))
 
             print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
                                                           len(attachments['audio_messages']),
@@ -517,7 +519,9 @@ def sortById(msg):
 
                 with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                     res = pool.starmap(copy_func(dmp._download),
-                                       zip(attachments['photos'], itertools.repeat(af)))
+                                       zip(itertools.repeat(dmp.__class__),
+                                           attachments['photos'],
+                                           itertools.repeat(af)))
 
                 print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
                                                               len(attachments['photos']),
@@ -542,7 +546,9 @@ def sortById(msg):
                 try:
                     with Pool(dmp._AVAILABLE_THREADS if dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp._settings['POOL_PROCESSES']) as pool:
                         res = pool.starmap(copy_func(dmp._download_video),
-                                           zip(videos['items'], itertools.repeat(af)))
+                                           zip(itertools.repeat(dmp.__class__),
+                                               videos['items'],
+                                               itertools.repeat(af)))
                 except MaybeEncodingError:
                     None
 
@@ -559,7 +565,9 @@ def sortById(msg):
 
                 with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                     res = pool.starmap(copy_func(dmp._download),
-                                       zip(attachments['docs'], itertools.repeat(af)))
+                                       zip(itertools.repeat(dmp.__class__),
+                                           attachments['docs'],
+                                           itertools.repeat(af)))
 
                 print('\x1b[2K      {}/{} (total: {})'.format(sum(filter(None, res)),
                                                               len(attachments['docs']),

diff --git a/modules/photo.py b/modules/photo.py
@@ -36,7 +36,8 @@ def dump_photo(dmp):
             print('    .../{}'.format(photo['count']), end='\r')
             with Pool(dmp._settings['POOL_PROCESSES']) as pool:
                 res = pool.starmap(copy_func(dmp._download),
-                                   zip(map(lambda p: sorted(p['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
+                                   zip(itertools.repeat(dmp.__class__),
+                                       map(lambda p: sorted(p['sizes'], key=itemgetter('width', 'height'))[-1]['url'], photo['items']),
                                        itertools.repeat(folder)))
 
             print('\x1b[2K    {}/{} (total: {})'.format(sum(filter(None, res)),

diff --git a/modules/video.py b/modules/video.py
@@ -41,7 +41,9 @@ def dump_video(dmp):
             print('    .../{}'.format(len(video['items'])), end='\r')
             with Pool(dmp._AVAILABLE_THREADS if dmp._settings['LIMIT_VIDEO_PROCESSES'] else dmp._settings['POOL_PROCESSES']) as pool:
                 res = pool.starmap(copy_func(dmp._download_video),
-                                   zip(video['items'], itertools.repeat(folder)))
+                                   zip(itertools.repeat(dmp.__class__),
+                                       video['items'],
+                                       itertools.repeat(folder)))
 
             print('\x1b[2K    {}/{} (total: {})'.format(sum(filter(None, res)),
                                                         len(video['items']),