Merge pull request #177 from jwoglom/custom-download-folder

Custom download folders (fixes #129)

README.md

@@ -36,6 +36,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __UMASK__: umask value used by MeTube. Defaults to `022`.
 * __DOWNLOAD_DIR__: path to where the downloads will be saved. Defaults to `/downloads` in the docker image, and `.` otherwise.
 * __AUDIO_DOWNLOAD_DIR__: path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`.
+* __CUSTOM_DIRS__: whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a drop-down appears next to the Add button to specify the download directory. Defaults to `true`.
+* __CREATE_CUSTOM_DIRS__: whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector becomes supports free-text input, and the specified directory will be created recursively. Defaults to `false`.
 * __STATE_DIR__: path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the docker image, and `.` otherwise.
 * __URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
 * __OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.

app/main.py

@@ -7,6 +7,7 @@
 import socketio
 import logging
 import json
+import pathlib
 
 from ytdl import DownloadQueueNotifier, DownloadQueue
 
@@ -16,6 +17,8 @@ class Config:
     _DEFAULTS = {
         'DOWNLOAD_DIR': '.',
         'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',
+        'CUSTOM_DIRS': 'true',
+        'CREATE_CUSTOM_DIRS': 'true',
         'STATE_DIR': '.',
         'URL_PREFIX': '',
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
@@ -80,7 +83,8 @@ async def add(request):
     if not url or not quality:
         raise web.HTTPBadRequest()
     format = post.get('format')
-    status = await dqueue.add(url, quality, format)
+    folder = post.get('folder')
+    status = await dqueue.add(url, quality, format, folder)
     return web.Response(text=serializer.encode(status))
 
 @routes.post(config.URL_PREFIX + 'delete')
@@ -96,6 +100,40 @@ async def delete(request):
 @sio.event
 async def connect(sid, environ):
     await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
+    await sio.emit('configuration', serializer.encode(config), to=sid)
+    if config.CUSTOM_DIRS:
+        await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
+
+def get_custom_dirs():
+    def recursive_dirs(base):
+        path = pathlib.Path(base)
+
+        # Converts PosixPath object to string, and remove base/ prefix
+        def convert(p):
+            s = str(p)
+            if s.startswith(base):
+                s = s[len(base):]
+
+            if s.startswith('/'):
+                s = s[1:]
+
+            return s
+
+        # Recursively lists all subdirectories of DOWNLOAD_DIR
+        dirs = list(filter(None, map(convert, path.glob('**'))))
+
+        return dirs
+
+    download_dir = recursive_dirs(config.DOWNLOAD_DIR)
+
+    audio_download_dir = download_dir
+    if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
+        audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
+
+    return {
+        "download_dir": download_dir,
+        "audio_download_dir": audio_download_dir
+    }
 
 @routes.get(config.URL_PREFIX)
 def index(request):
@@ -112,8 +150,15 @@ def index_redirect_dir(request):
 
 routes.static(config.URL_PREFIX + 'favicon/', 'favicon')
 routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR)
+routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR)
 routes.static(config.URL_PREFIX, 'ui/dist/metube')
-app.add_routes(routes)
+try:
+    app.add_routes(routes)
+except ValueError as e:
+    if 'ui/dist/metube' in str(e):
+        raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e
+    raise e
+
 
 
 # https://github.com/aio-libs/aiohttp/pull/4615 waiting for release

app/ytdl.py

@@ -28,10 +28,11 @@ async def cleared(self, id):
         raise NotImplementedError
 
 class DownloadInfo:
-    def __init__(self, id, title, url, quality, format):
+    def __init__(self, id, title, url, quality, format, folder):
         self.id, self.title, self.url = id, title, url
         self.quality = quality
         self.format = format
+        self.folder = folder
         self.status = self.msg = self.percent = self.speed = self.eta = None
         self.filename = None
         self.timestamp = time.time_ns()
@@ -197,7 +198,7 @@ def __init__(self, config, notifier):
 
     async def __import_queue(self):
         for k, v in self.queue.saved_items():
-            await self.add(v.url, v.quality, v.format)
+            await self.add(v.url, v.quality, v.format, folder=v.folder)
 
     async def initialize(self):
         self.event = asyncio.Event()
@@ -212,7 +213,7 @@ def __extract_info(self, url):
             **self.config.YTDL_OPTIONS,
         }).extract_info(url, download=False)
 
-    async def __add_entry(self, entry, quality, format, already):
+    async def __add_entry(self, entry, quality, format, folder, already):
         etype = entry.get('_type') or 'video'
         if etype == 'playlist':
             entries = entry['entries']
@@ -225,14 +226,28 @@ async def __add_entry(self, entry, quality, format, already):
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
                         etr[f"playlist_{property}"] = entry[property]
-                results.append(await self.__add_entry(etr, quality, format, already))
+                results.append(await self.__add_entry(etr, quality, format, folder, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
             return {'status': 'ok'}
         elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:
             if not self.queue.exists(entry['id']):
-                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format)
-                dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
+                dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder)
+                # Keep consistent with frontend
+                base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
+                if folder:
+                    if self.config.CUSTOM_DIRS != 'true':
+                        return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
+                    dldirectory = os.path.realpath(os.path.join(base_directory, folder))
+                    real_base_directory = os.path.realpath(base_directory)
+                    if not dldirectory.startswith(real_base_directory):
+                        return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
+                    if not os.path.isdir(dldirectory):
+                        if self.config.CREATE_CUSTOM_DIRS != 'true':
+                            return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'}
+                        os.makedirs(dldirectory, exist_ok=True)
+                else:
+                    dldirectory = base_directory
                 output = self.config.OUTPUT_TEMPLATE
                 output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
                 for property, value in entry.items():
@@ -243,11 +258,11 @@ async def __add_entry(self, entry, quality, format, already):
                 await self.notifier.added(dl)
             return {'status': 'ok'}
         elif etype.startswith('url'):
-            return await self.add(entry['url'], quality, format, already)
+            return await self.add(entry['url'], quality, format, folder, already)
         return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
 
-    async def add(self, url, quality, format, already=None):
-        log.info(f'adding {url}')
+    async def add(self, url, quality, format, folder, already=None):
+        log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}')
         already = set() if already is None else already
         if url in already:
             log.info('recursion detected, skipping')
@@ -258,7 +273,7 @@ async def add(self, url, quality, format, already=None):
             entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
-        return await self.__add_entry(entry, quality, format, already)
+        return await self.__add_entry(entry, quality, format, folder, already)
 
     async def cancel(self, ids):
         for id in ids:

ui/angular.json

@@ -30,7 +30,9 @@
             "styles": [
               "src/styles.sass"
             ],
-            "scripts": []
+            "scripts": [
+              "node_modules/bootstrap/dist/js/bootstrap.min.js",
+            ]
           },
           "configurations": {
             "production": {

ui/package.json

@@ -25,6 +25,7 @@
     "@fortawesome/free-regular-svg-icons": "^6.1.1",
     "@fortawesome/free-solid-svg-icons": "^6.1.1",
     "@ng-bootstrap/ng-bootstrap": "^12.0.0",
+    "@ng-select/ng-select": "^8.3.0",
     "bootstrap": "^5.0.0",
     "ngx-cookie-service": "^13.0.0",
     "ngx-socket-io": "^4.2.0",

ui/src/app/app.component.html

@@ -47,10 +47,21 @@
           </div>
         </div>
         <div class="col-md-3 add-url-component">
-          <button class="btn btn-primary add-url" type="submit" (click)="addDownload()" [disabled]="addInProgress || downloads.loading">
-            <span class="spinner-border spinner-border-sm" role="status" id="add-spinner" *ngIf="addInProgress"></span>
-            {{ addInProgress ? "Adding..." : "Add" }}
-          </button>
+          <div [attr.class]="showAdvanced() ? 'btn-group add-url-group' : 'add-url-group'" ngbDropdown #advancedDropdown="ngbDropdown" display="dynamic" placement="bottom-end">
+            <button class="btn btn-primary add-url" type="submit" (click)="addDownload()" [disabled]="addInProgress || downloads.loading">
+              <span class="spinner-border spinner-border-sm" role="status" id="add-spinner" *ngIf="addInProgress"></span>
+              {{ addInProgress ? "Adding..." : "Add" }}
+            </button>
+            <button class="btn btn-primary dropdown-toggle dropdown-toggle-split" id="advancedButton" type="button" title="Advanced options" [disabled]="addInProgress || downloads.loading" ngbDropdownAnchor (focus)="advancedDropdown.open()" *ngIf="showAdvanced()">
+              <span class="sr-only">Advanced options</span>
+            </button>
+            <div ngbDropdownMenu aria-labelledby="advancedButton" class="dropdown-menu dropdown-menu-end folder-dropdown-menu">
+              <div class="input-group">
+                <span class="input-group-text">Download Folder</span>
+                <ng-select [items]="customDirs$ | async" placeholder="Default" [addTag]="allowCustomDir.bind(this)" addTagText="Create directory" [ngStyle]="{'flex-grow':'1'}" bindLabel="folder" [(ngModel)]="folder" [disabled]="addInProgress || downloads.loading"></ng-select>
+              </div>
+            </div>
+          </div>
         </div>
       </div>
     </div>
@@ -118,11 +129,11 @@
             <fa-icon *ngIf="download.value.status == 'finished'" [icon]="faCheckCircle" style="color: green;"></fa-icon>
             <fa-icon *ngIf="download.value.status == 'error'" [icon]="faTimesCircle" style="color: red;"></fa-icon>
           </div>
-          <span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="download/{{download.value.filename | encodeURIComponent}}" target="_blank">{{ download.value.title }}</a></span>
+          <span ngbTooltip="{{download.value.msg}}"><a *ngIf="!!download.value.filename; else noDownloadLink" href="{{buildDownloadLink(download.value)}}" target="_blank">{{ download.value.title }}</a></span>
           <ng-template #noDownloadLink>{{ download.value.title }}</ng-template>
         </td>
         <td>
-          <button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
+          <button *ngIf="download.value.status == 'error'" type="button" class="btn btn-link" (click)="retryDownload(download.key, download.value.url, download.value.quality, download.value.folder)"><fa-icon [icon]="faRedoAlt"></fa-icon></button>
         </td>
         <td>
           <a href="{{download.value.url}}" target="_blank"><fa-icon [icon]="faExternalLinkAlt"></fa-icon></a>

ui/src/app/app.component.sass

@@ -9,9 +9,20 @@
 .add-url-component
     margin: 0.5rem auto
 
+.add-url-group
+    width: 100%
+
 button.add-url
     width: 100%
 
+.folder-dropdown-menu
+    width: 500px
+
+.folder-dropdown-menu .input-group
+    display: flex
+    padding-left: 5px
+    padding-right: 5px
+
 $metube-section-color-bg: rgba(0,0,0,.07)
 
 .metube-section-header