Added removal of downloads that are missing files

.gitignore

@@ -3,4 +3,5 @@ __pycache__/
 .vscode/
 config/config.conf
 ToDo 
-snip*.py
+snip*.py
+venv

README.md

@@ -13,12 +13,16 @@ Feature overview:
 You may run this locally by launching main.py, or by pulling the docker image.
 You can find a sample docker-compose.yml in the docker folder.
 
+## Dependencies
+Use Sonarr v4 & Radarr v5 (currently 'nightly' tag instead of 'latest'), else certain features may not work correctly.
+Use latest version of qBittorrent.
+
 ## Getting started
 There's two ways to run this:
 - As a docker container with docker-compose
 - By cloning the repository and running the script manually
 
-Both ways are explained below and there's an explanation for the different settings below that
+Both ways are explained below and there's an explanation for the different settings below that.
 
 ## Docker
 1) Make a `docker-compose.yml` file
@@ -67,8 +71,9 @@ services:
 ## Running manually
 1) Clone the repository with `git clone https://github.com/Fxsch/decluttarr.git`
 2) Rename the `config.conf-Example` inside the config folder to `config.conf`
-3) Tweak `config.conf` to your needs 
-4) Run the script with `python3 main.py`
+3) Tweak `config.conf` to your needs
+4) Install the libraries listed in the docker/requirements.txt (pip install -r requirements.txt)
+5) Run the script with `python3 main.py`
 Note: The `config.conf` is disregarded when running via docker-compose.yml
 
 ## Explanation of the settings
@@ -137,6 +142,13 @@ Note: The `config.conf` is disregarded when running via docker-compose.yml
 - Permissible Values: True, False
 - Is Mandatory: No (Defaults to False)
 
+**REMOVE_MISSING_FILES**
+- Steers whether downloads that have the warning "Files Missing" are removed from the queue
+- These downloads are not added to the blocklist
+- Type: Boolean
+- Permissible Values: True, False
+- Is Mandatory: No (Defaults to False)
+
 **REMOVE_SLOW**
 - Steers whether slow downloads are removed from the queue
 - Slow downloads are added to the blocklist, so that they are not re-requested in the future

config/config.conf-Example

@@ -9,7 +9,9 @@ REMOVE_STALLED              = True
 REMOVE_METADATA_MISSING     = True
 REMOVE_ORPHANS              = True
 REMOVE_UNMONITORED          = True
+REMOVE_MISSING_FILES        = True
 REMOVE_SLOW                 = True
+REMOVE_MISSING_FILES        = True
 MIN_DOWNLOAD_SPEED          = 100
 PERMITTED_ATTEMPTS          = 3
 NO_STALLED_REMOVAL_QBIT_TAG = Don't Kill

config/config.py

@@ -86,6 +86,7 @@ def get_config_value(key, config_section, is_mandatory, datatype, default_value
 REMOVE_METADATA_MISSING     = get_config_value('REMOVE_METADATA_MISSING',       'features',     False,  bool,   False)
 REMOVE_ORPHANS              = get_config_value('REMOVE_ORPHANS' ,               'features',     False,  bool,   False)
 REMOVE_UNMONITORED          = get_config_value('REMOVE_UNMONITORED' ,           'features',     False,  bool,   False)
+REMOVE_MISSING_FILES        = get_config_value('REMOVE_MISSING_FILES' ,         'features',     False,  bool,   False)
 REMOVE_SLOW                 = get_config_value('REMOVE_SLOW' ,                  'features',     False,  bool,   False)
 MIN_DOWNLOAD_SPEED          = get_config_value('MIN_DOWNLOAD_SPEED',            'features',     False,  int,    0)
 PERMITTED_ATTEMPTS          = get_config_value('PERMITTED_ATTEMPTS',            'features',     False,  int,    3)

main.py

@@ -159,6 +159,6 @@ async def main():
                 {settings_dict['SONARR_URL']: {}} if settings_dict['SONARR_URL'] else {} + \
                 {settings_dict['LIDARR_URL']: {}} if settings_dict['LIDARR_URL'] else {} 
     defective_tracker = Defective_Tracker(instances)
-    download_sizes = Download_Sizes()
+    download_sizes = Download_Sizes({})
     asyncio.run(main())
 

src/queue_cleaner.py

@@ -143,6 +143,22 @@ async def remove_unmonitored(settings_dict, BASE_URL, API_KEY, deleted_downloads
     logger.debug('remove_unmonitored/queue OUT: %s', str(await get_queue(BASE_URL, API_KEY) ))
     return len(unmonitoredItems)
 
+async def remove_missing_files(settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker):
+    # Detects downloads stuck downloading meta data and triggers repeat check and subsequent delete. Adds to blocklist  
+    queue = await get_queue(BASE_URL, API_KEY)
+    if not queue: return 0    
+    logger.debug('remove_missing_files/queue: %s', str(queue))
+    missing_filesItems = []
+    for queueItem in queue['records']:
+        if 'errorMessage' in queueItem and 'status' in queueItem:
+            if  queueItem['status']        == 'warning' and \
+                queueItem['errorMessage']  == 'DownloadClientQbittorrentTorrentStateMissingFiles':
+                    missing_filesItems.append(queueItem)
+    for queueItem in missing_filesItems:
+        await remove_download(settings_dict, BASE_URL, API_KEY, queueItem['id'], queueItem['title'], queueItem['downloadId'], 'missing files', False, deleted_downloads) 
+    logger.debug('remove_missing_files/queue OUT: %s', str(await get_queue(BASE_URL, API_KEY) ))
+    return len(missing_filesItems)
+
 async def remove_slow(settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker, download_sizes):
     # Detects slow downloads and triggers delete. Adds to blocklist
     queue = await get_queue(BASE_URL, API_KEY)
@@ -169,12 +185,10 @@ async def remove_slow(settings_dict, BASE_URL, API_KEY, deleted_downloads, defec
                             logger.verbose('>>> Detected slow download, tagged not to be killed: %s (%dKB/s)',queueItem['title'], speed)
                     else:
                         slowItems.append(queueItem)
-            try:
-                logger.verbose(f'{(downloaded_size - download_sizes.dict[queueItem["downloadId"]]) * settings_dict["REMOVE_TIMER"] / 60},  : {queueItem["title"]}')
-            except: pass
             download_sizes.dict[queueItem['downloadId']] = downloaded_size
     await check_permitted_attempts(settings_dict, slowItems, 'slow', True, deleted_downloads, BASE_URL, API_KEY, defective_tracker)
     queue = await get_queue(BASE_URL, API_KEY) 
+    logger.debug('remove_slow/download_sizes.dict: %s', str(download_sizes.dict))
     logger.debug('remove_slow/queue OUT: %s', str(queue))
     return len(slowItems)
 
@@ -185,14 +199,18 @@ async def check_permitted_attempts(settings_dict, current_defective_items, failT
     for queueItem in current_defective_items:
         current_defective[queueItem['id']] = {'title': queueItem['title'],'downloadId': queueItem['downloadId']}
     logger.debug('check_permitted_attempts/current_defective: %s', str(current_defective))
+
     # 2. Check if those that were previously defective are no longer defective -> those are recovered
     try:
         recovered_ids = [tracked_id for tracked_id in defective_tracker.dict[BASE_URL][failType] if tracked_id not in current_defective]
     except KeyError:
         recovered_ids = []
+
     logger.debug('check_permitted_attempts/recovered_ids: %s' + str(recovered_ids))
+
     for recovered_id in recovered_ids:
        del defective_tracker.dict[BASE_URL][failType][recovered_id]
+
     logger.debug('check_permitted_attempts/defective_tracker.dict IN: %s', str(defective_tracker.dict))
     # 3. For those that are defective, add attempt + 1 if present before, or make attempt = 0. If exceeding number of permitted attempts, delete hem
     download_ids_stuck = []
@@ -221,7 +239,7 @@ async def remove_download(settings_dict, BASE_URL, API_KEY, queueId, queueTitle,
     return
 
 ########### MAIN FUNCTION ###########
-async def queue_cleaner(settings_dict, arr_type, defective_tracker):
+async def queue_cleaner(settings_dict, arr_type, defective_tracker, download_sizes):
     # Read out correct instance depending on radarr/sonarr flag
     run_dict = {}
     if arr_type == 'radarr':
@@ -272,6 +290,9 @@ async def queue_cleaner(settings_dict, arr_type, defective_tracker):
         if settings_dict['REMOVE_UNMONITORED']: 
             items_detected += await remove_unmonitored(       settings_dict, BASE_URL, API_KEY, deleted_downloads, arr_type)
 
+        if settings_dict['REMOVE_MISSING_FILES']: 
+            items_detected += await remove_missing_files(     settings_dict, BASE_URL, API_KEY, deleted_downloads, arr_type)
+
         if settings_dict['REMOVE_SLOW']:
             items_detected += await remove_slow(              settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker, download_sizes)