diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7499dc2..af94c98 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,6 +21,7 @@ jobs: target key: "${{runner.os}} Rust ${{steps.rust-toolchain.outputs.cachekey}} Lock ${{hashFiles('Cargo.lock')}}" - uses: msrd0/install-rlottie-action@v1 + - run: sudo apt-get update -y && sudo apt-get install -y libavc1394-dev libavdevice-dev - run: cargo test --workspace --all-features --release -- --include-ignored env: RUST_BACKTRACE: 1 diff --git a/Cargo.lock b/Cargo.lock index 6efc33b..463835a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", +] + [[package]] name = "bindgen" version = "0.66.1" @@ -393,6 +413,31 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "ffmpeg-next" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8af03c47ad26832ab3aabc4cdbf210af3d3b878783edd5a7ba044ba33aab7a60" +dependencies = [ + "bitflags 1.3.2", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf650f461ccf130f4eef4927affed703cc387b183bfc4a7dfee86a076c131127" +dependencies = [ + "bindgen 0.64.0", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + [[package]] name = "flate2" version = "1.0.27" @@ -762,7 +807,7 @@ dependencies = [ "gif", "rgb", "rlottie", - "webp-animation", + "webp-animation 0.7.0", ] [[package]] @@ -861,6 +906,7 @@ dependencies = [ "anyhow", "async-trait", "derive-getters", + "ffmpeg-next", "flate2", "futures-util", "generic-array", @@ -881,6 +927,7 @@ dependencies = [ "tokio", "tokio-stream", "url", + "webp-animation 0.8.0", ] [[package]] @@ -1189,7 +1236,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "596a589bc1142eb480f662d2132f2188e45d09aee22fde2683465572b399fe03" dependencies = [ - "bindgen", + "bindgen 0.66.1", "pkg-config", ] @@ -1759,6 +1806,16 @@ dependencies = [ "log", ] +[[package]] +name = "webp-animation" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6999b9294c72c4c0000f5429caa2ab6f3cd7135b4d22ecfb01e779f3f9a3ed" +dependencies = [ + "libwebp-sys2", + "log", +] + [[package]] name = "webpki-roots" version = "0.25.2" diff --git a/mstickerlib/Cargo.toml b/mstickerlib/Cargo.toml index 15e9a31..ff7d024 100644 --- a/mstickerlib/Cargo.toml +++ b/mstickerlib/Cargo.toml @@ -15,6 +15,7 @@ include = ["/src/**/*.rs", "/LICENSE", "/README.md"] anyhow = "1.0" async-trait = "0.1" derive-getters = "0.3.0" +ffmpeg = { package = "ffmpeg-next", version = "6.0" } flate2 = "1.0" futures-util = "0.3.25" generic-array = { version = "0.14" , features = ["serde"] } @@ -34,6 +35,7 @@ tempfile = "3.2" tokio = { version = "1.21", features = ["fs", "parking_lot", "sync"] } tokio-stream = { version = "0.1", features = ["io-util"], default-features = false } url = "2.2" +webp-animation = "0.8" [dev-dependencies] tokio = { version = "1.21", features = ["macros"] } diff --git a/mstickerlib/src/image.rs b/mstickerlib/src/image.rs index 867f246..71ae23c 100644 --- a/mstickerlib/src/image.rs +++ b/mstickerlib/src/image.rs @@ -1,6 +1,7 @@ use crate::{ database, - matrix::{self, Config, Mxc} + matrix::{self, Config, Mxc}, + video::webm2webp }; use anyhow::anyhow; use flate2::write::GzDecoder; @@ -32,6 +33,19 @@ pub struct Image { pub height: u32 } +fn rayon_run(callback: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, + for<'a> &'a mut T: Send +{ + let mut result: Option = None; + rayon::scope(|s| { + s.spawn(|_| result = Some(callback())); + }); + result.unwrap() +} + impl Image { pub fn mime_type(&self) -> anyhow::Result { Ok(format!( @@ -56,18 +70,6 @@ impl Image { if !self.file_name.ends_with(".tgs") { return Ok(self); } - fn rayon_run(callback: F) -> T - where - F: FnOnce() -> T + Send, - T: Send, - for<'a> &'a mut T: Send - { - let mut result: Option = None; - rayon::scope(|s| { - s.spawn(|_| result = Some(callback())); - }); - result.unwrap() - } tokio::task::spawn_blocking(move || { rayon_run(move || { @@ -102,6 +104,38 @@ impl Image { .await? } + pub async fn convert_webm_if_webp(self, animation_format: Option) -> anyhow::Result { + match animation_format { + Some(AnimationFormat::Webp) => self.convert_webm2webp().await, + _ => Ok(self) + } + } + + /// convert `webm` video stickers to webp, ignore other formats + pub async fn convert_webm2webp(mut self) -> anyhow::Result { + if !self.file_name.ends_with(".webm") { + return Ok(self); + } + + tokio::task::spawn_blocking(move || { + rayon_run(move || { + let mut tmp = tempfile::Builder::new().suffix(".webm").tempfile()?; + tmp.write_all(&self.data)?; + tmp.flush()?; + + self.file_name.truncate(self.file_name.len() - 1); + self.file_name += "p"; + let (webp, width, height) = webm2webp(&tmp.path())?; + self.data = Arc::new(webp.to_vec()); + self.width = width; + self.height = height; + + Ok(self) + }) + }) + .await? + } + ///upload image to matrix /// return mxc_url and true if image was uploaded now; false if it was already uploaded before and exist at the database pub async fn upload(&self, matrix_config: &Config, database: Option<&D>) -> anyhow::Result<(Mxc, bool)> diff --git a/mstickerlib/src/lib.rs b/mstickerlib/src/lib.rs index 03c2ad3..56d46e4 100644 --- a/mstickerlib/src/lib.rs +++ b/mstickerlib/src/lib.rs @@ -8,6 +8,7 @@ pub mod database; pub mod image; pub mod matrix; pub mod tg; +mod video; //mod sub_commands; @@ -26,6 +27,13 @@ struct Client { client: UnsafeCell> } +// XXX Hacky: We abuse the fact that a client will be exactly once either set or +// created, so we can ensure this function will be called exactly once. Also, the +// HTTP client will always be needed before ffmpeg. +fn init() { + ffmpeg::init().expect("Failed to initialise ffmpeg"); +} + impl Client { pub(crate) async fn get(&self) -> &reqwest::Client { // safety: this method ensures that the client is set from None to Some exactly once, and the @@ -44,10 +52,12 @@ impl Client { let client = unsafe { self.client.get().as_mut().unwrap() }; if client.is_none() { *client = Some(reqwest::Client::new()); + init(); } client.as_ref().unwrap() } } + pub async fn set_client(client: reqwest::Client) { #[allow(unused_variables)] let guard = CLIENT.lock.read(); @@ -56,6 +66,7 @@ pub async fn set_client(client: reqwest::Client) { panic!("reqwest client was already set") } *lib_client = Some(client); + init(); } pub async fn get_client() -> &'static reqwest::Client { diff --git a/mstickerlib/src/tg/sticker.rs b/mstickerlib/src/tg/sticker.rs index 3a22b6d..6cfc95a 100644 --- a/mstickerlib/src/tg/sticker.rs +++ b/mstickerlib/src/tg/sticker.rs @@ -6,7 +6,6 @@ use crate::{ matrix::{self, sticker_formats::ponies, Mxc}, CLIENT }; -use anyhow::bail; use derive_getters::Getters; use log::warn; use serde::Deserialize; @@ -76,6 +75,8 @@ impl PhotoSize { .download(tg_config) .await? .convert_tgs_if_some(advance_config.animation_format) + .await? + .convert_webm_if_webp(advance_config.animation_format) .await?; #[cfg(feature = "log")] info!(" upload sticker{thumb:<10} {pack_name}:{positon:02} {emoji}"); @@ -129,16 +130,6 @@ impl Sticker { where D: crate::database::Database { - if self.is_video { - #[cfg(feature = "log")] - info!( - " skip Sticker {}:{:02} {}, is a video", - self.pack_name, - self.positon, - self.emoji.as_deref().unwrap_or_default() - ); - bail!("sticker is video") - } #[cfg(feature = "log")] info!( "download sticker {}:{:02} {}", diff --git a/mstickerlib/src/tg/stickerpack.rs b/mstickerlib/src/tg/stickerpack.rs index 945819b..afe4905 100644 --- a/mstickerlib/src/tg/stickerpack.rs +++ b/mstickerlib/src/tg/stickerpack.rs @@ -47,13 +47,6 @@ impl StickerPack { { #[cfg(feature = "log")] info!("import Telegram stickerpack {}({})", self.title, self.name); - #[cfg(feature = "log")] - if self.is_video { - warn!( - "sticker pack {} includes video stickers. Import of video stickers is not supported and will be skipped.", - self.name - ); - } let stickers_import_futures = self .stickers @@ -163,4 +156,10 @@ mod tests { async fn import_none() { import("NSanimated", None).await; } + + #[tokio::test] + #[ignore] + async fn import_video_pack_webp() { + import("pingu_animated", Some(AnimationFormat::Webp)).await; + } } diff --git a/mstickerlib/src/video.rs b/mstickerlib/src/video.rs new file mode 100644 index 0000000..85ed2f5 --- /dev/null +++ b/mstickerlib/src/video.rs @@ -0,0 +1,62 @@ +//! This module deals with translating telegram's video stickers to webp animations. + +use ffmpeg::{ + codec::Context as CodecContext, + decoder, + format::{self, Pixel}, + media::Type, + software::scaling::{context::Context as ScalingContext, flag::Flags}, + util::frame::video::Video +}; +use std::path::Path; +use webp_animation::{Encoder, WebPData}; + +pub(crate) fn webm2webp>(file: &P) -> anyhow::Result<(WebPData, u32, u32)> { + // heavily inspired by + // https://github.com/zmwangx/rust-ffmpeg/blob/master/examples/dump-frames.rs + + let mut ictx = format::input(file)?; + let input = ictx.streams().best(Type::Video).ok_or(ffmpeg::Error::StreamNotFound)?; + + let video_stream_index = input.index(); + let ctx_decoder = CodecContext::from_parameters(input.parameters())?; + let mut decoder = ctx_decoder.decoder().video()?; + + let mut scaler = ScalingContext::get( + decoder.format(), + decoder.width(), + decoder.height(), + Pixel::RGBA, + decoder.width(), + decoder.height(), + Flags::BILINEAR + )?; + + let mut encoder = Encoder::new((decoder.width(), decoder.height()))?; + let mut timestamp = 0; + let frame_rate = input.rate(); + let time_per_frame = frame_rate.1 * 1000 / frame_rate.0; + let mut receive_and_process_decoded_frames = |decoder: &mut decoder::Video| -> anyhow::Result<()> { + let mut decoded = Video::empty(); + while decoder.receive_frame(&mut decoded).is_ok() { + let mut rgba_frame = Video::empty(); + scaler.run(&decoded, &mut rgba_frame)?; + + encoder.add_frame(rgba_frame.data(0), timestamp)?; + timestamp += time_per_frame; + } + Ok(()) + }; + + for (stream, packet) in ictx.packets() { + if stream.index() == video_stream_index { + decoder.send_packet(&packet)?; + receive_and_process_decoded_frames(&mut decoder)?; + } + } + decoder.send_eof()?; + receive_and_process_decoded_frames(&mut decoder)?; + + let webp = encoder.finalize(timestamp)?; + Ok((webp, decoder.width(), decoder.height())) +}