ugoira re-encode and export local database option

PixivBookmarkHandler.py

@@ -304,6 +304,31 @@ def export_image_bookmark(caller,
         PixivHelper.print_and_log('error', 'Error at export_image_bookmark(): {0}'.format(sys.exc_info()))
         raise
 
+def export_image_table(caller, filename, pixiv, fanbox, sketch):
+    export_list = list()
+    table = list()
+    try:
+        if pixiv == 'o':
+            table.append("Pixiv")
+        elif fanbox == 'o':
+            table.append("Fanbox")
+        elif sketch == 'o':
+            table.append("Sketch")
+        else:
+            if pixiv == 'y':
+                table.append("Pixiv")
+            if fanbox == 'y':
+                table.append("Fanbox")
+            if sketch == 'y':
+                table.append("Sketch")
+        for t in table:
+            export_list = caller.__dbManager__.exportImageTable(t)
+            PixivBookmark.export_image_list(export_list, f"{filename}-{t}")
+    except KeyboardInterrupt:
+        raise
+    except BaseException:
+        PixivHelper.print_and_log('error', 'Error at export_image_table(): {0}'.format(sys.exc_info()))
+        raise
 
 def get_bookmarks(caller, config, hide, start_page=1, end_page=0, member_id=None):
     br: PixivBrowser = caller.__br__

PixivDBManager.py

@@ -217,6 +217,54 @@ def importList(self, listTxt):
         print('done.')
         return 0
 
+    def exportImageTable(self, name):
+        print(f'Exporting {name} table ...', end=' ')
+        im_list = list()
+        if name == "Pixiv":
+            table = "pixiv_master_image"
+            key = "image_id"
+        elif name == "Fanbox":
+            table = "fanbox_master_post"
+            key = "post_id"
+        elif name == "Sketch":
+            table = "sketch_master_post"
+            key = "post_id"
+        else:
+            raise
+        try:
+            c = self.conn.cursor()
+            c.execute(f''' SELECT COUNT(*) FROM {table}''')
+            result = c.fetchall()
+            if result[0][0] > 10000:
+                print('Row count is more than 10000 (actual row count:',
+                      str(result[0][0]), ')')
+                print('It may take a while to retrieve the data.')
+                arg = input('Continue [y/n, default is yes]').rstrip("\r") or 'y'
+                answer = arg.lower()
+                if answer not in ('y', 'n', 'o'):
+                    PixivHelper.print_and_log("error", f"Invalid args for TODO: {arg}, valid values are [y/n/o].")
+                    return
+                if answer == 'y':
+                    c = self.conn.cursor()
+                    c.execute(f'''SELECT {key}
+                                FROM {table}
+                                ORDER BY member_id''')
+                    for row in c:
+                        im_list.append(row[0])
+            else:
+                c.execute(f'''SELECT {key}
+                            FROM {table}
+                            ORDER BY member_id''')
+                for row in c:
+                        im_list.append(row[0])
+            c.close()
+            print('done.')
+            return im_list
+        except BaseException:
+            print('Error at exportImageTable():', str(sys.exc_info()))
+            print('failed')
+            raise
+
     def exportList(self, filename, include_artist_token=True):
         print('Exporting list...', end=' ')
         try:

PixivImageHandler.py

@@ -437,3 +437,65 @@ def process_manga_series(caller,
         PixivHelper.print_and_log('error', f'Error at process_manga_series(): {manga_series_id}')
         PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')
         raise
+
+def process_ugoira_local(caller, config):
+    directory = config.rootDirectory
+    all_directory = list()
+    all_zip = list()
+    counter = 0
+
+    for folder in os.scandir(directory):
+        folder_name = os.path.basename(folder)
+        if folder_name != ".DS_Store":
+            all_directory.append(folder_name)
+    try:
+        # Go through all directory
+        for sub_directory in all_directory:
+            sub_directory = os.scandir(directory+"/"+sub_directory)
+            for file in sub_directory:
+                # Get name of the file and it s extension
+
+                    file_name = os.path.splitext(os.path.basename(file))[0]
+                    file_ext = os.path.splitext(os.path.basename(file))[1]
+                    # Check zip archive
+                    if file_ext == ".zip":
+                        counter += 1
+                        PixivHelper.print_and_log(None, f"\r{counter} zip files found", newline = False)
+                        all_zip.append(os.path.abspath(file))
+
+        print('')
+        nb_zip = counter
+        counter = 0
+        for zip in all_zip:
+            counter += 1
+            PixivHelper.print_and_log(None, f"# Ugoira {counter}/{nb_zip}")
+            zip_name = os.path.splitext(os.path.basename(zip))[0]
+            PixivHelper.print_and_log("info", f"Deleting old ugoira files ...", newline = False)
+            for file in os.scandir(os.path.dirname(zip)):
+                file_name = os.path.splitext(os.path.basename(file))[0]
+                file_ext = os.path.splitext(os.path.basename(file))[1]
+                if (file_name == zip_name):
+                    if  ((("ugoira" in file_ext) and (config.createUgoira))    or
+                        ((("gif" in file_ext) and (config.createGif)))         or
+                        (("apng" in file_ext) and (config.createApng))         or
+                        (("webm" in file_ext) and (config.createWebm))         or
+                        (("webp" in file_ext) and (config.createWebp))):
+                        PixivHelper.print_and_log("debug", f"Deleting {os.path.abspath(file)}")
+                        os.remove(os.path.abspath(file))
+            PixivHelper.print_and_log(None, f" done.")
+            # Get id artwork
+            image_id = ''
+            for letter in zip_name:
+                if letter == '_':
+                    break
+                image_id = image_id+letter
+            process_image(  caller,
+                            config,
+                            artist=None,
+                            image_id=image_id,
+                            useblacklist=False)
+
+    except Exception as ex:
+        PixivHelper.print_and_log('error', 'Error at process_ugoira_local(): %s' %str(sys.exc_info()))
+        PixivHelper.print_and_log('error', 'failed')
+        raise

PixivUtil2.py

@@ -182,10 +182,12 @@ def menu():
     print(' b. Batch Download from batch_job.json (experimental)')
     print(Style.BRIGHT + '── Others '.ljust(PADDING, "─") + Style.RESET_ALL)
     print(' d. Manage database')
+    print(' l. Export local database.')
     print(' e. Export online followed artist.')
     print(' m. Export online other\'s followed artist.')
     print(' p. Export online image bookmarks.')
     print(' i. Import list file')
+    print(' u. Ugoira re-encode')
     print(' r. Reload config.ini')
     print(' p. Print config.ini')
     print(' x. Exit')
@@ -729,6 +731,65 @@ def menu_download_by_group_id(opisvalid, args, options):
                                             limit=limit,
                                             process_external=process_external)
 
+def menu_ugoira_reencode(opisvalid, args, options):
+
+    msg = Fore.YELLOW + Style.NORMAL + f'WARNING: THIS ACTION CANNOT BE UNDO !' + Style.RESET_ALL
+    PixivHelper.print_and_log(None, msg)
+    msg = Fore.YELLOW + Style.NORMAL + f'You are about to to re-encode and overwrite all of your ugoira based on its zip file.' + Style.RESET_ALL
+    PixivHelper.print_and_log(None, msg)
+    arg = input(Fore.YELLOW + Style.BRIGHT + 'Do you really to proceed ? [y/n, default is no]: ' + Style.RESET_ALL).rstrip("\r") or 'n'
+    sure = arg.lower()
+    if sure not in ('y', 'n'):
+        PixivHelper.print_and_log("error", f"Invalid args for ugoira reencode: {arg}, valid values are [y/n].")
+        return
+    if sure == 'y':
+        PixivImageHandler.process_ugoira_local(sys.modules[__name__], __config__)
+
+def menu_export_database_images(opisvalid, args, options):
+    __log__.info('Export local database')
+    use_pixiv = "n"  # y|n|o
+    use_fanbox = "n" # y|n|o
+    use_sketch = "n" # y|n|o
+    filename = "export-database.txt"
+
+    if opisvalid:
+        if options.export_filename is not None:
+            filename = options.export_filename
+        if options.use_pixiv is not None:
+            use_pixiv = options.use_pixiv
+            if use_pixiv not in ('y', 'n', 'o'):
+                PixivHelper.print_and_log("error", f"Invalid args for Pixiv database: {use_pixiv}, valid values are [y/n/o].")
+                return
+        if options.use_fanbox is not None:
+            use_fanbox = options.use_fanbox
+            if use_fanbox not in ('y', 'n', 'o'):
+                PixivHelper.print_and_log("error", f"Invalid args for Fanbox database: {use_fanbox}, valid values are [y/n/o].")
+                return
+        if options.use_sketch is not None:
+            use_sketch = options.use_sketch
+            if use_sketch not in ('y', 'n', 'o'):
+                PixivHelper.print_and_log("error", f"Invalid args for Sketch database: {use_sketch}, valid values are [y/n/o].")
+                return
+
+    else:
+        filename = input("Filename: ").rstrip("\r") or filename
+        arg = input("Include Pixiv database [y/n/o, default is no]: ").rstrip("\r") or 'n'
+        use_pixiv = arg.lower()
+        if use_pixiv not in ('y', 'n', 'o'):
+            PixivHelper.print_and_log("error", f"Invalid args for Fanbox database: {arg}, valid values are [y/n/o].")
+            return
+        arg = input("Include Fanbox database [y/n/o, default is no]: ").rstrip("\r") or 'n'
+        use_fanbox = arg.lower()
+        if use_fanbox not in ('y', 'n', 'o'):
+            PixivHelper.print_and_log("error", f"Invalid args for Fanbox database: {arg}, valid values are [y/n/o].")
+            return
+        arg = input("Include Sketch database [y/n/o, default is no]: ").rstrip("\r") or 'n'
+        use_sketch = arg.lower()
+        if use_sketch not in ('y', 'n', 'o'):
+            PixivHelper.print_and_log("error", f"Invalid args for Sketch database: {arg}, valid values are [y/n/o].")
+            return
+
+    PixivBookmarkHandler.export_image_table(sys.modules[__name__], filename, use_pixiv, use_fanbox, use_sketch)
 
 def menu_export_online_bookmark(opisvalid, args, options):
     __log__.info('Export Followed Artists mode (e).')
@@ -1136,7 +1197,7 @@ def setup_option_parser():
     __valid_options = ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18',
                        'f1', 'f2', 'f3', 'f4', 'f5',
                        's1', 's2',
-                       'd', 'e', 'm', 'b', 'p')
+                       'l', 'd', 'e', 'm', 'b', 'p')
     parser = OptionParser()
 
     # need to keep the whitespace to adjust the output for --help
@@ -1162,6 +1223,7 @@ def setup_option_parser():
 s1 - Download by creator id (Sketch)')              \n
 s2 - Download by post id (Sketch)')                 \n
 b  - Batch Download from batch_job.json             \n
+l  - Export local database (image_id)               \n
 e  - Export online bookmark                         \n
 m  - Export online user bookmark                    \n
 p  - Export online image bookmark                   \n
@@ -1275,6 +1337,27 @@ def setup_option_parser():
                       default="export.txt",
                       help='''Filename for exporting members/images.                    \n
 Used in option e, m, p''')
+    parser.add_option('--up', '--use_pixiv',
+                      dest='use_pixiv',
+                      default=None,
+                      help='''Use Pixiv table for export.                               \n
+ y - include pixiv database.                        \n
+ n - don't include pixiv database.                     \n
+ o - only export pixiv database.''')
+    parser.add_option('--uf', '--use_fanbox',
+                      dest='use_fanbox',
+                      default=None,
+                      help='''Use Fanbox table for export.                              \n
+ y - include fanbox database.                       \n
+ n - don't include fanbox database.                 \n
+ o - only export fanbox database.''')
+    parser.add_option('--us', '--use_sketch',
+                      dest='use_sketch',
+                      default=None,
+                      help='''Use Sketch table for export.                              \n
+ y - include sketch database.                       \n
+ n - don't include sketch database.                 \n
+ o - only export sketch database.''')
     return parser
 
 
@@ -1334,6 +1417,8 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid_local, args, options):
                 menu_download_by_rank_r18(op_is_valid, args, options)
             elif selection == '18':
                 menu_download_new_illusts(op_is_valid, args, options)
+            elif selection == "l":
+                menu_export_database_images(op_is_valid, args, options)
             elif selection == 'b':
                 PixivBatchHandler.process_batch_job(sys.modules[__name__], batch_file=options.batch_file)
             elif selection == 'e':
@@ -1342,6 +1427,8 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid_local, args, options):
                 menu_export_online_user_bookmark(op_is_valid, args, options)
             elif selection == 'p':
                 menu_export_from_online_image_bookmark(op_is_valid, args, options)
+            elif selection == 'u':
+                menu_ugoira_reencode(op_is_valid, args, options)
             elif selection == 'd':
                 __dbManager__.main()
             elif selection == 'r':
@@ -1350,6 +1437,7 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid_local, args, options):
                 menu_print_config()
             elif selection == 'i':
                 menu_import_list()
+
             # PIXIV FANBOX
             elif selection == 'f1':
                 menu_fanbox_download_from_list(op_is_valid, PixivModelFanbox.FanboxArtist.SUPPORTING, args, options)

readme.md

@@ -30,13 +30,15 @@
 - Download by artist/creator id (FANBOX)
 - Download by post id (FANBOX)
 - Download from followed artists (FANBOX)
+- Re-encoding of all ugoira present in folder
 - Batch Download from batch_job.json (experimental)
   See https://github.com/Nandaka/PixivUtil2/wiki/Using-Batch-Job-(Experimental)
 - Manage database:
   - Show all member
   - Show all downloaded images
   - Export list (member_id only)
   - Export list (detailed)
+  - Export local database (image_id)
   - Show member by last downloaded date
   - Show image by image_id
   - Show member by member_id
@@ -265,7 +267,7 @@ Please refer run with `--help` for latest information.
                             (required: Group ID, limit, and process external[y/n])
                         13 - Download by Manga Series ID
                             (required: Manga Series ID separated by space
-                             optional: --sp=START_PAGE, and --ep=END_PAGE))
+                            optional: --sp=START_PAGE, and --ep=END_PAGE)
                         f1 - Download from supported artists (FANBOX)
                             (optional: End Page)
                         f2 - Download by artist/creator id (FANBOX)
@@ -279,6 +281,8 @@ Please refer run with `--help` for latest information.
                             (optional: End page, path to list)
                         b - Batch Download from batch_job.json (experimental)
                             (optional: --bf=BATCH_FILE)
+                        l - Export local database image_id/post_id
+                            (required: --up=USE_PIXIV, and --uf=USE_FANBOX, and --us=USE_SKETCH)
                         e - Export online bookmark
                             (required: -p BOOKMARK_FLAG [y/n/o] for private bookmark
                              optional: filename)