Tzahi12345 · Tzahi12345 · Jun 27, 2022 · Jun 18, 2022 · Jun 19, 2022 · Jun 19, 2022
diff --git a/Dockerfile b/Dockerfile
@@ -47,11 +47,11 @@ RUN npm config set strict-ssl false && \
 
 # Final image
 FROM base
-RUN npm install -g pm2 && \
-    apt update && \
-    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 atomicparsley && \
+RUN apt update && \
+    apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
+RUN pip install tcd
 WORKDIR /app
 # User 1000 already exist from base image
 COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ]
@@ -64,4 +64,4 @@ RUN chmod +x /app/fix-scripts/*.sh
 
 EXPOSE 17442
 ENTRYPOINT [ "/app/entrypoint.sh" ]
-CMD [ "pm2-runtime","--raw","pm2.config.js" ]
+CMD [ "npm","start" ]
diff --git a/Dockerfile.heroku b/Dockerfile.heroku
@@ -1,2 +1,2 @@
-FROM tzahi12345/youtubedl-material:nightly
-CMD [ "pm2-runtime", "pm2.config.js" ]
+FROM tzahi12345/youtubedl-material:latest
+CMD [ "npm", "start" ]
diff --git a/Public API v1.yaml b/Public API v1.yaml
@@ -97,6 +97,11 @@ paths:
       summary: Get all files
       description: Gets all files and playlists stored in the db
       operationId: get-getAllFiles
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GetAllFilesRequest'
       responses:
         '200':
           description: OK
@@ -1724,6 +1729,41 @@ components:
           description: All video playlists
           items:
             $ref: '#/components/schemas/Playlist'
+    GetAllFilesRequest:
+      type: object
+      properties:
+        sort:
+          $ref: '#/components/schemas/Sort'
+        range:
+          type: array
+          items:
+            type: number
+            description: Two elements allowed, start index and end index
+          minItems: 2
+          maxItems: 2
+        text_search:
+          type: string
+          description: Filter files by title
+        file_type_filter:
+          $ref: '#/components/schemas/FileTypeFilter'
+        sub_id:
+          type: string
+          description: Include if you want to filter by subscription
+    Sort:
+      type: object
+      properties:
+        by:
+          type: string
+          description: Property to sort by
+        order:
+          type: number
+          description: 1 for ascending, -1 for descending
+    FileTypeFilter:
+      type: string
+      enum:
+        - audio_only
+        - video_only
+        - both
     GetAllFilesResponse:
       required:
         - files
@@ -1786,7 +1826,6 @@ components:
       required:
         - name
         - url
-        - streamingOnly
       type: object
       properties:
         name:
@@ -1899,7 +1938,6 @@ components:
         - uids
         - playlistName
         - thumbnailURL
-        - type
       type: object
       properties:
         playlistName:
@@ -1908,8 +1946,6 @@ components:
           type: array
           items:
             type: string
-        type:
-          $ref: '#/components/schemas/FileType'
         thumbnailURL:
           type: string
     CreatePlaylistResponse:
@@ -1939,15 +1975,17 @@ components:
       required:
         - playlist
         - success
-        - type
       type: object
       properties:
         playlist:
           $ref: '#/components/schemas/Playlist'
-        type:
-          $ref: '#/components/schemas/FileType'
         success:
           type: boolean
+        file_objs:
+          type: array
+          description: File objects for every uid in the playlist's uids property, in the same order
+          items:
+            $ref: '#/components/schemas/DatabaseFile'
     GetPlaylistsRequest:
       type: object
       properties:
@@ -1972,13 +2010,10 @@ components:
     DeletePlaylistRequest:
       required:
         - playlist_id
-        - type
       type: object
       properties:
         playlist_id:
           type: string
-        type:
-          $ref: '#/components/schemas/FileType'
     DownloadFileRequest:
       type: object
       properties:
@@ -2385,6 +2420,16 @@ components:
           type: number
         local_view_count:
           type: number
+        sub_id:
+          type: string
+        registered:
+          type: number
+        height:
+          type: number
+          description: In pixels, only for videos
+        abr:
+          type: number
+          description: In Kbps
     Playlist:
       required:
         - uids
@@ -2466,6 +2511,8 @@ components:
           type: string
         sub_name:
           type: string
+        prefetched_info:
+          type: object
     Task:
       required:
         - key
@@ -2480,6 +2527,8 @@ components:
       properties:
         key:
           type: string
+        title:
+          type: string
         last_ran:
           type: number
         last_confirmed:
@@ -2560,7 +2609,6 @@ components:
         - url
         - type
         - user_uid
-        - streamingOnly
         - isPlaylist
         - videos
       type: object
@@ -2576,8 +2624,6 @@ components:
         user_uid:
           type: string
           nullable: true
-        streamingOnly:
-          type: boolean
         isPlaylist:
           type: boolean
         archive:

diff --git a/README.md b/README.md
@@ -12,16 +12,6 @@ Now with [Docker](#Docker) support!
 
 <hr>
 
-### USAGE OF THE NIGHTLY BUILDS IS HIGHLY RECOMMENDED.
-
-For much better scaling with large datasets please run your YTDL-M instance with a MongoDB backend rather than the json file-based default.
-It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
-The (closed) issues as well as the project's Wiki will give you good starting points for your journey!
-
-For MongoDB specifically there is [this little guide](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
-
-<hr>
-
 ## Getting Started
 
 Check out the prerequisites, and go to the installation section. Easy as pie!
@@ -58,6 +48,7 @@ sudo yum install nodejs youtube-dl ffmpeg ffmpeg-devel
 Optional dependencies:
 
 * AtomicParsley (for embedding thumbnails, package name `atomicparsley`)
+* [tcd](https://github.com/PetterKraabol/Twitch-Chat-Downloader) (for downloading Twitch VOD chats)
 
 ### Installing
 
@@ -91,7 +82,7 @@ Alternatively, you can port forward the port specified in the config (defaults t
 
 ### Host-specific instructions
 
-If you're on a Synology NAS, unRAID or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
+If you're on a Synology NAS, unRAID, Raspberry Pi 4 or any other possible special case you can check if there's known issues or instructions both in the issue tracker and in the [Wiki!](https://github.com/Tzahi12345/YoutubeDL-Material/wiki#environment-specific-guideshelp)
 
 ### Setup
 
@@ -102,8 +93,6 @@ If you are looking to setup YoutubeDL-Material with Docker, this section is for
 3. Run `docker-compose up` to start it up. If successful, it should say "HTTP(S): Started on port 17443" or something similar. This tells you the *container-internal* port of the application. Please check your `docker-compose.yml` file for the *external* port. If you downloaded the file as described above, it defaults to **8998**.
 4. Make sure you can connect to the specified URL + *external* port, and if so, you are done!
 
-NOTE: It is currently recommended that you use the `nightly` tag on Docker. To do so, simply update the docker-compose.yml `image` field so that it points to `tzahi12345/youtubedl-material:nightly`.
-
 ### Custom UID/GID
 
 By default, the Docker container runs as non-root with UID=1000 and GID=1000. To set this to your own UID/GID, simply update the `environment` section in your `docker-compose.yml` like so:
@@ -114,6 +103,12 @@ environment:
     GID: YOUR_GID
 ```
 
+## MongoDB
+
+For much better scaling with large datasets please run your YoutubeDL-Material instance with MongoDB backend rather than the json file-based default. It will fix a lot of performance problems (especially with datasets in the tens of thousands videos/audios)!
+
+[Tutorial](https://github.com/Tzahi12345/YoutubeDL-Material/wiki/Setting-a-MongoDB-backend-to-use-as-database-provider-for-YTDL-M).
+
 ## API
 
 [API Docs](https://youtubedl-material.stoplight.io/docs/youtubedl-material/Public%20API%20v1.yaml)

diff --git a/SECURITY.md b/SECURITY.md
@@ -2,16 +2,16 @@
 
 ## Supported Versions
 
-Currently all work on this project goes into the nightly builds.
-4.2's RELEASE build is now quite old and should be considered legacy.
-We urge users to use the nightly releases, because the project
-constantly sees fixes.
+If you would like to see the latest updates, use the `nightly` tag on Docker.
 
-| Version       | Supported          |
-| ------------- | ------------------ |
-| 4.2 Nightlies | :white_check_mark: |
-| 4.2 Release   | :x:                |
-| < 4.2         | :x:                |
+If you'd like to stick with more stable releases, use the `latest` tag on Docker or download the [latest release here](https://github.com/Tzahi12345/YoutubeDL-Material/releases/latest). 
+
+| Version              | Supported          |
+| -------------------- | ------------------ |
+| 4.3 Docker Nightlies | :white_check_mark: |
+| 4.3 Release          | :white_check_mark: |
+| 4.2 Release          | :x:                |
+| < 4.2                | :x:                |
 
 ## Reporting a Vulnerability
 

diff --git a/backend/app.js b/backend/app.js
@@ -101,7 +101,6 @@ let backendPort = null;
 let useDefaultDownloadingAgent = null;
 let customDownloadingAgent = null;
 let allowSubscriptions = null;
-let archivePath = path.join(__dirname, 'appdata', 'archives');
 
 // other needed values
 let url_domain = null;
@@ -500,12 +499,13 @@ async function loadConfig() {
     loadConfigValues();
 
     // connect to DB
-    await db_api.connectToDB();
+    if (!config_api.getConfigItem('ytdl_use_local_db'))
+        await db_api.connectToDB();
     db_api.database_initialized = true;
     db_api.database_initialized_bs.next(true);
 
     // creates archive path if missing
-    await fs.ensureDir(archivePath);
+    await fs.ensureDir(utils.getArchiveFolder());
 
     // check migrations
     await checkMigrations();
@@ -912,11 +912,11 @@ app.post('/api/getFile', optionalJwt, async function (req, res) {
 app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
     let files = null;
-    let playlists = null;
-    let sort = req.body.sort;
-    let range = req.body.range;
-    let text_search = req.body.text_search;
-    let file_type_filter = req.body.file_type_filter;
+    const sort = req.body.sort;
+    const range = req.body.range;
+    const text_search = req.body.text_search;
+    const file_type_filter = req.body.file_type_filter;
+    const sub_id = req.body.sub_id;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
     const filter_obj = {user_uid: uuid};
@@ -929,6 +929,10 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
         }
     }
 
+    if (sub_id) {
+        filter_obj['sub_id'] = sub_id;
+    }
+
     if (file_type_filter === 'audio_only') filter_obj['isAudio'] = true;
     else if (file_type_filter === 'video_only') filter_obj['isAudio'] = false;
 
@@ -1268,7 +1272,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
     subscription = JSON.parse(JSON.stringify(subscription));
 
     // get sub videos
-    if (subscription.name && !subscription.streamingOnly) {
+    if (subscription.name) {
         var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
         subscription['videos'] = parsed_files;
         // loop through files for extra processing
@@ -1278,19 +1282,6 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
             if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
         }
 
-        res.send({
-            subscription: subscription,
-            files: parsed_files
-        });
-    } else if (subscription.name && subscription.streamingOnly) {
-        // return list of videos
-        let parsed_files = [];
-        if (subscription.videos) {
-            for (let i = 0; i < subscription.videos.length; i++) {
-                const video = subscription.videos[i];
-                parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date, video.view_count, video.height, video.abr));
-            }
-        }
         res.send({
             subscription: subscription,
             files: parsed_files
@@ -1335,9 +1326,8 @@ app.post('/api/getSubscriptions', optionalJwt, async (req, res) => {
 app.post('/api/createPlaylist', optionalJwt, async (req, res) => {
     let playlistName = req.body.playlistName;
     let uids = req.body.uids;
-    let type = req.body.type;
 
-    const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null);
+    const new_playlist = await db_api.createPlaylist(playlistName, uids, req.isAuthenticated() ? req.user.uid : null);
 
     res.send({
         new_playlist: new_playlist,
@@ -1365,7 +1355,6 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     res.send({
         playlist: playlist,
         file_objs: file_objs,
-        type: playlist && playlist.type,
         success: !!playlist
     });
 });

diff --git a/backend/appdata/default.json b/backend/appdata/default.json
@@ -31,7 +31,8 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
       "generate_NFO_files": false
@@ -63,7 +64,7 @@
       "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib"
     },
     "Advanced": {
-      "default_downloader": "youtube-dl",
+      "default_downloader": "yt-dlp",
       "use_default_downloading_agent": true,
       "custom_downloading_agent": "",
       "multi_user_mode": false,