Add video support to the caption and meme commands

.env.example

@@ -48,7 +48,19 @@ METRICS=
 # The image API type to be used
 # Set this to `none` to process all images locally
 # Set this to `ws` if you want to use the external image API script, located in api/index.js
+# NOTE: VIDEOS WILL BE PROCESSED LOCALLY REGARDLESS
 API_TYPE=none
 
 # Put ID of server to limit owner-only commands to
-ADMIN_SERVER=
+ADMIN_SERVER=
+
+# Maximum number of media processing operations that can be running concurrently at any given time.
+MEDIA_CONCURRENCY_LIMIT=1
+
+# Maximum size in pixels for each dimension of an output video
+MAX_VIDEO_DIMENSIONS=1024
+# Stop processing a video after the output reaches the following size
+# (The size of the output file is slightly bigger than this value.)
+MAX_VIDEO_SIZE=64Mi
+# Stop processing a video after the following time (in milliseconds) has elapsed
+VIDEO_TIMEOUT=60000

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "assets/images/region-flags"]
 	path = assets/images/region-flags
 	url = https://github.com/fonttools/region-flags
+[submodule "ffmpeg"]
+	path = ffmpeg
+	url = https://github.com/dd-pardal/esmBot-FFmpeg.git

CMakeLists.txt

@@ -5,16 +5,24 @@ project(image)
 
 file(GLOB SOURCE_FILES "natives/*.cc" "natives/*.h")
 
-if (CMAKE_JS_VERSION)
+add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES})
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)
+
+if(CMAKE_JS_VERSION)
   include_directories(${CMAKE_JS_INC})
-  add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC} natives/node/image.cc)
-  set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
-  target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB})
+  add_library(${PROJECT_NAME}-node SHARED ${CMAKE_JS_SRC} natives/node/image.cc)
+  set_target_properties(${PROJECT_NAME}-node PROPERTIES PREFIX "" OUTPUT_NAME "image" SUFFIX ".node")
+  target_link_libraries(${PROJECT_NAME}-node ${CMAKE_JS_LIB} ${PROJECT_NAME})
+  target_compile_features(${PROJECT_NAME}-node PRIVATE cxx_std_17)
+
+  add_library(${PROJECT_NAME}-c SHARED natives/c-bindings/image.cc)
+  target_link_libraries(${PROJECT_NAME}-c ${PROJECT_NAME})
+  target_compile_features(${PROJECT_NAME}-c PRIVATE cxx_std_17)
 else()
-  add_executable(${PROJECT_NAME} ${SOURCE_FILES} natives/cli/image.cc)
+  add_executable(${PROJECT_NAME}-cli natives/cli/image.cc)
+  target_compile_features(${PROJECT_NAME}-cli PRIVATE cxx_std_17)
 endif()
 
-target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)
 
 if(MSVC) # todo: change flags for more parity with GCC/clang, I don't know much about MSVC so pull requests are open
   set(CMAKE_CXX_FLAGS "/Wall /EHsc /GS")

README.md

@@ -1,3 +1,9 @@
+# Video Support
+
+This is an experimental fork of esmBot with video support, powered by FFmpeg.
+
+If you want to try this out, follow te instructions in `docs/setup.md`. If you have already esmBot set up, you'll probably just need to install the required libraries (`libvpx-dev libopus-dev libssl-dev` if you're on Debian/Ubuntu), `git clone --recurse-submodules --depth 10` this repo, copy the `.env` file and run `pnpm run build`. The script will build everything, including FFmpeg.
+
 # <img src="https://github.com/esmBot/esmBot/raw/master/docs/assets/esmbot.png" width="128"> esmBot
 [![esmBot Support](https://discordapp.com/api/guilds/592399417676529688/embed.png)](https://discord.gg/esmbot) ![GitHub license](https://img.shields.io/github/license/esmBot/esmBot.svg)
 

api/index.js

@@ -11,7 +11,7 @@ import { createRequire } from "module";
 import EventEmitter from "events";
 
 const nodeRequire = createRequire(import.meta.url);
-const img = nodeRequire(`../build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/image.node`);
+const img = nodeRequire(`../natives/image/build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/image.node`);
 
 const Rerror = 0x01;
 const Tqueue = 0x02;

classes/mediaCommand.js

@@ -1,13 +1,27 @@
 import Command from "./command.js";
-import imageDetect from "../utils/imagedetect.js";
+import mediaDetect from "../utils/media-detection.js";
 import { runImageJob } from "../utils/image.js";
 import { runningCommands } from "../utils/collections.js";
 import { readFileSync } from "fs";
 const { emotes } = JSON.parse(readFileSync(new URL("../config/messages.json", import.meta.url)));
 import { random } from "../utils/misc.js";
 import { selectedImages } from "../utils/collections.js";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { readFile } from "fs/promises";
+import ConcurrencyLimiter from "../utils/concurrency-limiter.js";
 
-class ImageCommand extends Command {
+const execFileP = promisify(execFile);
+
+export const ffmpegConfig = {
+  // Pixel formats supported by the encoder, separated by "|"
+  pixelFormats: "yuv420p|yuva420p"
+};
+
+// const limiter = new ConcurrencyLimiter(Number.parseInt(process.env.MEDIA_CONCURRENCY_LIMIT));
+const limiter = new ConcurrencyLimiter(1);
+
+class MediaCommand extends Command {
   async criteria() {
     return true;
   }
@@ -30,29 +44,25 @@ class ImageCommand extends Command {
       },
       id: (this.interaction ?? this.message).id
     };
+    let media;
 
     if (this.type === "application") await this.acknowledge();
 
     if (this.constructor.requiresImage) {
       try {
         const selection = selectedImages.get(this.author.id);
-        const image = selection ?? await imageDetect(this.client, this.message, this.interaction, this.options, true);
+        media = selection ?? await mediaDetect(this.client, this.message, this.interaction, this.options, true, true);
         if (selection) selectedImages.delete(this.author.id);
-        if (image === undefined) {
+        if (media === undefined) {
           runningCommands.delete(this.author.id);
           return `${this.constructor.noImage} (Tip: try right-clicking/holding on a message and press Apps -> Select Image, then try again.)`;
-        } else if (image.type === "large") {
+        } else if (media.type === "large") {
           runningCommands.delete(this.author.id);
-          return "That image is too large (>= 25MB)! Try using a smaller image.";
-        } else if (image.type === "tenorlimit") {
+          return "That file is too large (>= 25MB)! Try using a smaller file.";
+        } else if (media.type === "tenorlimit") {
           runningCommands.delete(this.author.id);
           return "I've been rate-limited by Tenor. Please try uploading your GIF elsewhere.";
         }
-        imageParams.path = image.path;
-        imageParams.params.type = image.type;
-        imageParams.url = image.url; // technically not required but can be useful for text filtering
-        imageParams.name = image.name;
-        if (this.constructor.requiresGIF) imageParams.onlyGIF = true;
       } catch (e) {
         runningCommands.delete(this.author.id);
         throw e;
@@ -61,46 +71,98 @@ class ImageCommand extends Command {
 
     if (this.constructor.requiresText) {
       const text = this.options.text ?? this.args.join(" ").trim();
-      if (text.length === 0 || !await this.criteria(text, imageParams.url)) {
+      if (text.length === 0 || !await this.criteria(text, media.url)) {
         runningCommands.delete(this.author.id);
         return this.constructor.noText;
       }
     }
 
-    if (typeof this.params === "function") {
-      Object.assign(imageParams.params, this.params(imageParams.url, imageParams.name));
-    } else if (typeof this.params === "object") {
-      Object.assign(imageParams.params, this.params);
+    if (media.mediaType === "video" && !this.constructor.acceptsVideo) {
+      runningCommands.delete(this.author.id);
+      return `This command only supports ${this.constructor.requiresGIF ? "" : "images and "}GIFs.`;
     }
 
     let status;
-    if (imageParams.params.type === "image/gif" && this.type === "classic") {
+    if ((media.mediaType === "video" || media.type === "image/gif") && this.type === "classic") {
       status = await this.processMessage(this.message.channel ?? await this.client.rest.channels.get(this.message.channelID));
     }
+
+    return limiter.runWhenFree(async () => {
+      if (media.mediaType === "video") {
+        try {
+          const params = this.ffmpegParams(media.url);
+          // TODO: Stream the data from FFmpeg directly to Discord and to a file in TEMPDIR. If
+          // there is an error, abort the request and remove the file. If the request body exceeds
+          // 8MiB, abort the request and, when processing has finished, send the embed with the
+          // link to TMP_DOMAIN and keep the file. If there is no error and the request doesn't
+          // exceed 8MiB, remove the file.
+          const output = await execFileP("ffmpeg/ffmpeg", [
+            // Global options
+            "-y", "-nostats",
+            // Input
+            "-i", media.url,
+            // Filter
+            "-vf", `fps=25, scale='min(${process.env.MAX_VIDEO_DIMENSIONS},iw)':'min(${process.env.MAX_VIDEO_DIMENSIONS},ih)':force_original_aspect_ratio=decrease:sws_flags=fast_bilinear, ${params.filterGraph}, scale='min(${process.env.MAX_VIDEO_DIMENSIONS},iw)':'min(${process.env.MAX_VIDEO_DIMENSIONS},ih)':force_original_aspect_ratio=decrease:sws_flags=fast_bilinear`,
+            // Video encoding
+            "-c:v", "libvpx", "-minrate", "500k", "-b:v", "2000k", "-maxrate", "5000k", "-cpu-used", "5", "-auto-alt-ref", "0",
+            // Audio encoding
+            "-c:a", "libopus", "-b:a", "64000",
+            // Output
+            "-fs", process.env.MAX_VIDEO_SIZE, "-f", "webm", "/tmp/output.webm",
+          ], {
+            // TODO: Send a SIGKILL some seconds after the timeout in case FFmpeg hangs
+            timeout: Number.parseInt(process.env.VIDEO_TIMEOUT)
+          });
+          return {
+            contents: await readFile("/tmp/output.webm"),
+            name: "output.webm"
+          }
+        } finally {
+          try {
+            if (status) await status.delete();
+          } catch {
+            // no-op
+          }
+          runningCommands.delete(this.author.id);
+        }
+      } else {
+        imageParams.path = media.path;
+        imageParams.params.type = media.type;
+        imageParams.url = media.url; // technically not required but can be useful for text filtering
+        imageParams.name = media.name;
+        if (this.constructor.requiresGIF) imageParams.onlyGIF = true;
 
-    try {
-      const { buffer, type } = await runImageJob(imageParams);
-      if (type === "nogif" && this.constructor.requiresGIF) {
-        return "That isn't a GIF!";
-      }
-      this.success = true;
-      return {
-        contents: buffer,
-        name: `${this.constructor.command}.${type}`
-      };
-    } catch (e) {
-      if (e === "Request ended prematurely due to a closed connection") return "This image job couldn't be completed because the server it was running on went down. Try running your command again.";
-      if (e === "Job timed out" || e === "Timeout") return "The image is taking too long to process (>=15 minutes), so the job was cancelled. Try using a smaller image.";
-      if (e === "No available servers") return "I can't seem to contact the image servers, they might be down or still trying to start up. Please wait a little bit.";
-      throw e;
-    } finally {
-      try {
-        if (status) await status.delete();
-      } catch {
-        // no-op
+        if (typeof this.params === "function") {
+          Object.assign(imageParams.params, this.params(imageParams.url, imageParams.name));
+        } else if (typeof this.params === "object") {
+          Object.assign(imageParams.params, this.params);
+        }
+
+        try {
+          const { buffer, type } = await runImageJob(imageParams);
+          if (type === "nogif" && this.constructor.requiresGIF) {
+            return "That isn't a GIF!";
+          }
+          this.success = true;
+          return {
+            contents: buffer,
+            name: `${this.constructor.command}.${type}`
+          };
+        } catch (e) {
+          if (e === "Request ended prematurely due to a closed connection") return "This image job couldn't be completed because the server it was running on went down. Try running your command again.";
+          if (e === "Job timed out" || e === "Timeout") return "The image is taking too long to process (>=15 minutes), so the job was cancelled. Try using a smaller image.";
+          if (e === "No available servers") return "I can't seem to contact the image servers, they might be down or still trying to start up. Please wait a little bit.";
+          throw e;
+        } finally {
+          try {
+            if (status) await status.delete();
+          } catch {
+            // no-op
+          }
+          runningCommands.delete(this.author.id);
+        }
       }
-      runningCommands.delete(this.author.id);
-    }
+    });
 
   }
 
@@ -122,19 +184,19 @@ class ImageCommand extends Command {
     }
     if (this.requiresImage) {
       this.flags.push({
-        name: "image",
+        name: this.acceptsVideo ? "media" : "image",
         type: 11,
-        description: "An image/GIF attachment"
+        description: `An image/GIF${this.acceptsVideo ? "/video" : ""} attachment`
       }, {
         name: "link",
         type: 3,
-        description: "An image/GIF URL"
+        description: `An image/GIF${this.acceptsVideo ? "/video" : ""} URL`
       });
     }
     this.flags.push({
       name: "togif",
       type: 5,
-      description: "Force GIF output"
+      description: "Force GIF output if the input is an image"
     });
     return this;
   }
@@ -145,9 +207,10 @@ class ImageCommand extends Command {
   static requiresText = false;
   static textOptional = false;
   static requiresGIF = false;
+  static acceptsVideo = false;
   static noImage = "You need to provide an image/GIF!";
   static noText = "You need to provide some text!";
   static command = "";
 }
 
-export default ImageCommand;
+export default MediaCommand;

commands/fun/homebrew.js

@@ -1,7 +1,7 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 import { cleanMessage } from "../../utils/misc.js";
 
-class HomebrewCommand extends ImageCommand {
+class HomebrewCommand extends MediaCommand {
   params(url) {
     const newArgs = this.options.text ?? this.args.filter(item => !item.includes(url)).join(" ");
     return {

commands/fun/sonic.js

@@ -1,8 +1,8 @@
 //import wrap from "../../utils/wrap.js";
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 import { cleanMessage } from "../../utils/misc.js";
 
-class SonicCommand extends ImageCommand {
+class SonicCommand extends MediaCommand {
   params() {
     const cleanedMessage = cleanMessage(this.message ?? this.interaction, this.options.text ?? this.args.join(" "));
     return {

commands/general/qrread.js

@@ -3,7 +3,7 @@ import { request } from "undici";
 import sharp from "sharp";
 import { clean } from "../../utils/misc.js";
 import Command from "../../classes/command.js";
-import imageDetect from "../../utils/imagedetect.js";
+import imageDetect from "../../utils/media-detection.js";
 
 class QrReadCommand extends Command {
   async run() {

commands/general/raw.js

@@ -1,5 +1,5 @@
 import Command from "../../classes/command.js";
-import imageDetect from "../../utils/imagedetect.js";
+import imageDetect from "../../utils/media-detection.js";
 
 class RawCommand extends Command {
   async run() {

commands/general/sticker.js

@@ -1,5 +1,5 @@
 import Command from "../../classes/command.js";
-import imagedetect from "../../utils/imagedetect.js";
+import imagedetect from "../../utils/media-detection.js";
 
 class StickerCommand extends Command {
   async run() {

commands/media-editing/9gag.js

@@ -1,6 +1,6 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 
-class NineGagCommand extends ImageCommand {
+class NineGagCommand extends MediaCommand {
   params = {
     water: "assets/images/9gag.png",
     gravity: 6

commands/media-editing/avs4you.js

@@ -1,6 +1,6 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 
-class AVSCommand extends ImageCommand {
+class AVSCommand extends MediaCommand {
   params = {
     water: "assets/images/avs4you.png",
     gravity: 5,

commands/media-editing/bandicam.js

@@ -1,6 +1,6 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 
-class BandicamCommand extends ImageCommand {
+class BandicamCommand extends MediaCommand {
   params = {
     water: "assets/images/bandicam.png",
     gravity: 2,

commands/media-editing/blur.js

@@ -1,6 +1,6 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 
-class BlurCommand extends ImageCommand {
+class BlurCommand extends MediaCommand {
   params = {
     sharp: false
   };

commands/media-editing/bounce.js

@@ -1,6 +1,6 @@
-import ImageCommand from "../../classes/imageCommand.js";
+import MediaCommand from "../../classes/mediaCommand.js";
 
-class BounceCommand extends ImageCommand {
+class BounceCommand extends MediaCommand {
   static description = "Makes an image bounce up and down";
   static aliases = ["bouncy"];