From 974c829dbc7d27dd6965989edf611f566329b29d Mon Sep 17 00:00:00 2001 From: Lancer Date: Thu, 16 Apr 2026 00:02:33 +0800 Subject: [PATCH 1/5] [chore] Add diffusers-format example and seed parameter to LongCatAudioDiTPipeline Signed-off-by: Lancer --- .../en/api/pipelines/longcat_audio_dit.md | 28 ++++++++----------- .../pipeline_longcat_audio_dit.py | 26 +++++++++++++++++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/source/en/api/pipelines/longcat_audio_dit.md b/docs/source/en/api/pipelines/longcat_audio_dit.md index 86488416727e..5248933f14f5 100644 --- a/docs/source/en/api/pipelines/longcat_audio_dit.md +++ b/docs/source/en/api/pipelines/longcat_audio_dit.md @@ -14,15 +14,10 @@ specific language governing permissions and limitations under the License. LongCat-AudioDiT is a text-to-audio diffusion model from Meituan LongCat. The diffusers integration exposes a standard [`DiffusionPipeline`] interface for text-conditioned audio generation. -This pipeline supports loading the original flat LongCat checkpoint layout from either a local directory or a Hugging Face Hub repository containing: - -- `config.json` -- `model.safetensors` - -The loader builds the text encoder, transformer, and VAE from `config.json`, restores component weights from `model.safetensors`, and ties the shared UMT5 embedding when needed. - This pipeline was adapted from the LongCat-AudioDiT reference implementation: https://github.com/meituan-longcat/LongCat-AudioDiT +This pipeline supports loading from a local directory or Hugging Face Hub repository in diffusers format (containing `text_encoder/`, `transformer/`, `vae/`, `tokenizer/`, and `scheduler/` subfolders). + ## Usage ```py @@ -31,27 +26,28 @@ import torch from diffusers import LongCatAudioDiTPipeline pipeline = LongCatAudioDiTPipeline.from_pretrained( - "meituan-longcat/LongCat-AudioDiT-1B", + "ruixiangma/LongCat-AudioDiT-1B-Diffusers", torch_dtype=torch.float16, ) pipeline = pipeline.to("cuda") +prompt = "A calm ocean wave ambience with soft wind in the background." audio = pipeline( - prompt="A calm ocean wave ambience with soft wind in the background.", - audio_end_in_s=5.0, + prompt, + audio_duration_s=5.0, num_inference_steps=16, guidance_scale=4.0, - output_type="pt", -).audios + seed=42, +).audios[0, 0] -output = audio[0, 0].float().cpu().numpy() -sf.write("longcat.wav", output, pipeline.sample_rate) +sf.write("longcat.wav", audio, pipeline.sample_rate) ``` ## Tips -- `audio_end_in_s` is the most direct way to control output duration. -- `output_type="pt"` returns a PyTorch tensor shaped `(batch, channels, samples)`. +- `audio_duration_s` is the most direct way to control output duration. +- `seed` makes generation reproducible (optional, defaults to None). +- Output shape is `(batch, channels, samples)` - use `.audios[0, 0]` to get a single audio sample. ## LongCatAudioDiTPipeline diff --git a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py index 200f87d7b973..0c95054928c8 100644 --- a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py +++ b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py @@ -25,12 +25,29 @@ from ...models import LongCatAudioDiTTransformer, LongCatAudioDiTVae from ...schedulers import FlowMatchEulerDiscreteScheduler from ...utils import logging +from ...utils.doc_utils import replace_example_docstring from ...utils.torch_utils import randn_tensor from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline logger = logging.get_logger(__name__) +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import soundfile as sf + >>> import torch + >>> from diffusers import LongCatAudioDiTPipeline + + >>> pipe = LongCatAudioDiTPipeline.from_pretrained("ruixiangma/LongCat-AudioDiT-1B-Diffusers") + >>> pipe.to("cuda") + + >>> prompt = "A calm ocean wave ambience with soft wind in the background." + >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, seed=42).audios[0, 0] + >>> sf.write("output.wav", audio, pipe.sample_rate) + ``` +""" + def _lens_to_mask(lengths: torch.Tensor, length: int | None = None) -> torch.BoolTensor: if length is None: @@ -194,6 +211,7 @@ def check_inputs( ) @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) def __call__( self, prompt: str | list[str], @@ -202,6 +220,7 @@ def __call__( latents: torch.Tensor | None = None, num_inference_steps: int = 16, guidance_scale: float = 4.0, + seed: int | None = None, generator: torch.Generator | list[torch.Generator] | None = None, output_type: str = "np", return_dict: bool = True, @@ -220,6 +239,7 @@ def __call__( Pre-generated noisy latents of shape `(batch_size, duration, latent_dim)`. num_inference_steps (`int`, defaults to 16): Number of denoising steps. guidance_scale (`float`, defaults to 4.0): Guidance scale for classifier-free guidance. + seed (`int`, *optional*): A seed used to make generation deterministic. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): Random generator(s). output_type (`str`, defaults to `"np"`): Output format: `"np"`, `"pt"`, or `"latent"`. return_dict (`bool`, defaults to `True`): Whether to return `AudioPipelineOutput`. @@ -228,7 +248,13 @@ def __call__( inputs specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`list`, defaults to `["latents"]`): Tensor inputs passed to `callback_on_step_end`. + + Examples: """ + # Create generator from seed if provided + if generator is None and seed is not None: + generator = torch.Generator(device=self.device).manual_seed(seed) + if prompt is None: prompt = [] elif isinstance(prompt, str): From ac4ec51a66f7544e9668b7875d8069a8fe738bb5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Apr 2026 00:41:30 +0000 Subject: [PATCH 2/5] Apply style fixes --- .../pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py index 0c95054928c8..b0dd20b0a7bd 100644 --- a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py +++ b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py @@ -43,7 +43,9 @@ >>> pipe.to("cuda") >>> prompt = "A calm ocean wave ambience with soft wind in the background." - >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, seed=42).audios[0, 0] + >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, seed=42).audios[ + ... 0, 0 + ... ] >>> sf.write("output.wav", audio, pipe.sample_rate) ``` """ From 533d6b6c58f63728c431fb3fd8dc1f6a457b5373 Mon Sep 17 00:00:00 2001 From: Lancer <402430575@qq.com> Date: Thu, 16 Apr 2026 10:23:13 +0800 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: dg845 <58458699+dg845@users.noreply.github.com> --- docs/source/en/api/pipelines/longcat_audio_dit.md | 2 +- .../pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/en/api/pipelines/longcat_audio_dit.md b/docs/source/en/api/pipelines/longcat_audio_dit.md index 5248933f14f5..9acffaa2192d 100644 --- a/docs/source/en/api/pipelines/longcat_audio_dit.md +++ b/docs/source/en/api/pipelines/longcat_audio_dit.md @@ -37,7 +37,7 @@ audio = pipeline( audio_duration_s=5.0, num_inference_steps=16, guidance_scale=4.0, - seed=42, + generator=torch.Generator("cuda").manual_seed(42), ).audios[0, 0] sf.write("longcat.wav", audio, pipeline.sample_rate) diff --git a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py index b0dd20b0a7bd..215d357f64e8 100644 --- a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py +++ b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py @@ -222,7 +222,6 @@ def __call__( latents: torch.Tensor | None = None, num_inference_steps: int = 16, guidance_scale: float = 4.0, - seed: int | None = None, generator: torch.Generator | list[torch.Generator] | None = None, output_type: str = "np", return_dict: bool = True, From e2ac8bd2f9cf752b1a2355051f28467fa7a88bcc Mon Sep 17 00:00:00 2001 From: Lancer Date: Thu, 16 Apr 2026 10:25:35 +0800 Subject: [PATCH 4/5] upd Signed-off-by: Lancer --- docs/source/en/api/pipelines/longcat_audio_dit.md | 3 ++- .../longcat_audio_dit/pipeline_longcat_audio_dit.py | 9 +-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/source/en/api/pipelines/longcat_audio_dit.md b/docs/source/en/api/pipelines/longcat_audio_dit.md index 9acffaa2192d..4ecdbd39d356 100644 --- a/docs/source/en/api/pipelines/longcat_audio_dit.md +++ b/docs/source/en/api/pipelines/longcat_audio_dit.md @@ -46,8 +46,9 @@ sf.write("longcat.wav", audio, pipeline.sample_rate) ## Tips - `audio_duration_s` is the most direct way to control output duration. -- `seed` makes generation reproducible (optional, defaults to None). +- Use `generator=torch.Generator("cuda").manual_seed(42)` to make generation reproducible. - Output shape is `(batch, channels, samples)` - use `.audios[0, 0]` to get a single audio sample. +- The pipeline outputs mono audio (1 channel). If you need stereo, you can duplicate the channel: `audio.unsqueeze(0).repeat(1, 2, 1)`. ## LongCatAudioDiTPipeline diff --git a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py index 215d357f64e8..1169a28a6538 100644 --- a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py +++ b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py @@ -43,9 +43,7 @@ >>> pipe.to("cuda") >>> prompt = "A calm ocean wave ambience with soft wind in the background." - >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, seed=42).audios[ - ... 0, 0 - ... ] + >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, generator=torch.Generator("cuda").manual_seed(42)).audios[0, 0] >>> sf.write("output.wav", audio, pipe.sample_rate) ``` """ @@ -240,7 +238,6 @@ def __call__( Pre-generated noisy latents of shape `(batch_size, duration, latent_dim)`. num_inference_steps (`int`, defaults to 16): Number of denoising steps. guidance_scale (`float`, defaults to 4.0): Guidance scale for classifier-free guidance. - seed (`int`, *optional*): A seed used to make generation deterministic. generator (`torch.Generator` or `list[torch.Generator]`, *optional*): Random generator(s). output_type (`str`, defaults to `"np"`): Output format: `"np"`, `"pt"`, or `"latent"`. return_dict (`bool`, defaults to `True`): Whether to return `AudioPipelineOutput`. @@ -252,10 +249,6 @@ def __call__( Examples: """ - # Create generator from seed if provided - if generator is None and seed is not None: - generator = torch.Generator(device=self.device).manual_seed(seed) - if prompt is None: prompt = [] elif isinstance(prompt, str): From e73a6a6dc580e6c6e3310ab2c52ce7251436e179 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Apr 2026 04:23:54 +0000 Subject: [PATCH 5/5] Apply style fixes --- .../longcat_audio_dit/pipeline_longcat_audio_dit.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py index 1169a28a6538..e6478535b373 100644 --- a/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py +++ b/src/diffusers/pipelines/longcat_audio_dit/pipeline_longcat_audio_dit.py @@ -43,7 +43,13 @@ >>> pipe.to("cuda") >>> prompt = "A calm ocean wave ambience with soft wind in the background." - >>> audio = pipe(prompt, audio_duration_s=5.0, num_inference_steps=20, guidance_scale=4.0, generator=torch.Generator("cuda").manual_seed(42)).audios[0, 0] + >>> audio = pipe( + ... prompt, + ... audio_duration_s=5.0, + ... num_inference_steps=20, + ... guidance_scale=4.0, + ... generator=torch.Generator("cuda").manual_seed(42), + ... ).audios[0, 0] >>> sf.write("output.wav", audio, pipe.sample_rate) ``` """