From 9b6d51d192f3cda270b44b37e5864727576854cd Mon Sep 17 00:00:00 2001 From: lmz <1352359183@qq.com> Date: Sun, 30 Apr 2023 23:16:22 +0800 Subject: [PATCH 1/4] update --- audio-chatgpt.py | 8 +++----- download.sh | 8 ++------ requirements.txt | 12 ++++++++++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/audio-chatgpt.py b/audio-chatgpt.py index 7a45383..faaa4b1 100644 --- a/audio-chatgpt.py +++ b/audio-chatgpt.py @@ -4,8 +4,6 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'NeuralSeq')) sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'text_to_audio/Make_An_Audio')) -sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'text_to_audio/Make_An_Audio_img')) -sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'text_to_audio/Make_An_Audio_inpaint')) sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'audio_detection')) sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mono2binaural')) import gradio as gr @@ -217,7 +215,7 @@ class I2A: def __init__(self, device): print("Initializing Make-An-Audio-Image to %s" % device) self.device = device - self.sampler = self._initialize_model('text_to_audio/Make_An_Audio_img/configs/img_to_audio/img2audio_args.yaml', 'text_to_audio/Make_An_Audio_img/useful_ckpts/ta54_epoch=000216.ckpt', device=device) + self.sampler = self._initialize_model('text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml', 'text_to_audio/Make_An_Audio/useful_ckpts/ta54_epoch=000216.ckpt', device=device) self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio_img/vocoder/logs/bigv16k53w',device=device) def _initialize_model(self, config, ckpt, device): @@ -421,8 +419,8 @@ class Inpaint: def __init__(self, device): print("Initializing Make-An-Audio-inpaint to %s" % device) self.device = device - self.sampler = self._initialize_model_inpaint('text_to_audio/Make_An_Audio_inpaint/configs/inpaint/txt2audio_args.yaml', 'text_to_audio/Make_An_Audio_inpaint/useful_ckpts/inpaint7_epoch00047.ckpt') - self.vocoder = VocoderBigVGAN('./vocoder/logs/bigv16k53w',device=device) + self.sampler = self._initialize_model_inpaint('text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml', 'text_to_audio/Make_An_Audio/useful_ckpts/inpaint7_epoch00047.ckpt') + self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio/vocoder/logs/bigv16k53',device=device) self.cmap_transform = matplotlib.cm.viridis def _initialize_model_inpaint(self, config, ckpt): diff --git a/download.sh b/download.sh index dc0a627..c23cc9f 100644 --- a/download.sh +++ b/download.sh @@ -8,14 +8,10 @@ wget -P checkpoints/0109_hifigan_bigpopcs_hop128/ -i https://huggingface.co/spac wget -P checkpoints/0102_xiaoma_pe/ -i https://huggingface.co/spaces/Silentlin/DiffSinger/blob/main/checkpoints/0102_xiaoma_pe/config.yaml https://huggingface.co/spaces/Silentlin/DiffSinger/resolve/main/checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt # Text to audio cd text_to_audio -git clone https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio -git clone https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_img -git clone https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_inpaint wget -P text_to_audio/Make_An_Audio/useful_ckpts/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio/resolve/main/useful_ckpts/ta40multi_epoch=000085.ckpt wget -P text_to_audio/Make_An_Audio/useful_ckpts/CLAP/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio/resolve/main/useful_ckpts/CLAP/CLAP_weights_2022.pth -wget -P text_to_audio/Make_An_Audio_img/useful_ckpts/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_img/resolve/main/useful_ckpts/ta54_epoch=000216.ckpt -wget -P text_to_audio/Make_An_Audio_img/useful_ckpts/CLAP/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_img/blob/main/useful_ckpts/CLAP/CLAP_weights_2022.pth -wget -P text_to_audio/Make_An_Audio_inpaint/useful_ckpts/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_inpaint/resolve/main/useful_ckpts/inpaint7_epoch00047.ckpt +wget -P text_to_audio/Make_An_Audio/useful_ckpts/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_img/resolve/main/useful_ckpts/ta54_epoch=000216.ckpt +wget -P text_to_audio/Make_An_Audio/useful_ckpts/ -i https://huggingface.co/spaces/DiffusionSpeech/Make_An_Audio_inpaint/resolve/main/useful_ckpts/inpaint7_epoch00047.ckpt # Text to speech wget -P checkpoints/GenerSpeech/ -i https://huggingface.co/spaces/Rongjiehuang/GenerSpeech/blob/main/checkpoints/GenerSpeech/config.yaml https://huggingface.co/spaces/Rongjiehuang/GenerSpeech/resolve/main/checkpoints/GenerSpeech/model_ckpt_steps_300000.ckpt wget -P checkpoints/trainset_hifigan/ -i https://huggingface.co/spaces/Rongjiehuang/GenerSpeech/blob/main/checkpoints/trainset_hifigan/config.yaml https://huggingface.co/spaces/Rongjiehuang/GenerSpeech/resolve/main/checkpoints/trainset_hifigan/model_ckpt_steps_1000000.ckpt diff --git a/requirements.txt b/requirements.txt index f884d53..e5a8d4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ --extra-index-url https://download.pytorch.org/whl/cu113 accelerate addict==2.4.0 +aiofiles albumentations==1.3.0 appdirs==1.4.4 basicsr==1.4.2 @@ -10,17 +11,23 @@ diffusers einops==0.3.0 espnet espnet_model_zoo +ffmpeg-python g2p-en==2.1.0 google==3.0.0 gradio -h5py==2.8.0 +h5py imageio==2.9.0 imageio-ffmpeg==0.4.2 invisible-watermark>=0.1.5 +jieba kornia==0.6 langchain==0.0.101 librosa +loguru miditoolkit==0.1.7 +mmcv==1.5.0 +mmdet==2.23.0 +mmengine==0.7.2 moviepy==1.0.3 numpy==1.23.1 omegaconf==2.1.1 @@ -56,8 +63,9 @@ torchlibrosa torchmetrics==0.6.0 torchvision==0.13.1 transformers==4.26.1 -typing-extensions==3.10.0.2 +typing-extensions==4.0.0 uuid==1.30 webdataset==0.2.5 webrtcvad==2.0.10 yapf==0.32.0 +git+https://github.com/openai/CLIP.git \ No newline at end of file From 79fe509375d26db19800a0658492449f1571276d Mon Sep 17 00:00:00 2001 From: lmz <1352359183@qq.com> Date: Sun, 30 Apr 2023 23:20:22 +0800 Subject: [PATCH 2/4] update --- .../configs/img_to_audio/img2audio_args.yaml | 77 +++++++++ .../configs/inpaint/txt2audio_args.yaml | 68 ++++++++ .../configs/text_to_audio}/clap_args.yaml | 0 .../configs/text_to_audio}/hifigan_args.yaml | 0 .../text_to_audio}/txt2audio_args.yaml | 0 .../ldm/data/extract_mel_spectrogram.py | 151 ++++++++++++++++++ .../ldm/lr_scheduler.py | 0 .../ldm/models/autoencoder.py | 0 .../ldm/models/autoencoder_multi.py | 0 .../ldm/models/diffusion/__init__.py | 0 .../ldm/models/diffusion/classifier.py | 0 .../ldm/models/diffusion/ddim.py | 0 .../ldm/models/diffusion/ddpm.py | 0 .../ldm/models/diffusion/ddpm_audio.py | 0 .../models/diffusion/ddpm_audio_inpaint.py | 0 .../ldm/models/diffusion/plms.py | 0 .../ldm/modules/attention.py | 0 .../ldm/modules/diffusionmodules/__init__.py | 0 .../diffusionmodules/custom_openaimodel.py | 0 .../ldm/modules/diffusionmodules/model.py | 0 .../modules/diffusionmodules/openaimodel.py | 0 .../ldm/modules/diffusionmodules/util.py | 0 .../ldm/modules/discriminator/model.py | 0 .../discriminator/multi_window_disc.py | 0 .../ldm/modules/distributions/__init__.py | 0 .../modules/distributions/distributions.py | 0 .../ldm/modules/ema.py | 0 .../ldm/modules/encoders/CLAP/CLAPWrapper.py | 0 .../ldm/modules/encoders/CLAP/__init__.py | 0 .../ldm/modules/encoders/CLAP/audio.py | 0 .../ldm/modules/encoders/CLAP/clap.py | 0 .../ldm/modules/encoders/CLAP/config.yml | 0 .../ldm/modules/encoders/CLAP/utils.py | 0 .../ldm/modules/encoders/__init__.py | 0 .../ldm/modules/encoders/modules.py | 38 ++++- .../modules/encoders/open_clap/__init__.py | 0 .../ldm/modules/encoders/open_clap/bert.py | 0 .../open_clap/bpe_simple_vocab_16e6.txt.gz | Bin .../ldm/modules/encoders/open_clap/factory.py | 0 .../encoders/open_clap/feature_fusion.py | 0 .../ldm/modules/encoders/open_clap/htsat.py | 0 .../encoders/open_clap/linear_probe.py | 0 .../ldm/modules/encoders/open_clap/loss.py | 0 .../ldm/modules/encoders/open_clap/model.py | 0 .../open_clap/model_configs/HTSAT-base.json | 0 .../open_clap/model_configs/HTSAT-large.json | 0 .../model_configs/HTSAT-tiny-win-1536.json | 0 .../open_clap/model_configs/HTSAT-tiny.json | 0 .../open_clap/model_configs/PANN-10.json | 0 .../model_configs/PANN-14-fmax-18k.json | 0 .../model_configs/PANN-14-fmax-8k-20s.json | 0 .../PANN-14-tiny-transformer.json | 0 .../model_configs/PANN-14-win-1536.json | 0 .../open_clap/model_configs/PANN-14.json | 0 .../open_clap/model_configs/PANN-6.json | 0 .../model_configs/RN101-quickgelu.json | 0 .../open_clap/model_configs/RN101.json | 0 .../model_configs/RN50-quickgelu.json | 0 .../open_clap/model_configs/RN50.json | 0 .../open_clap/model_configs/RN50x16.json | 0 .../open_clap/model_configs/RN50x4.json | 0 .../open_clap/model_configs/ViT-B-16.json | 0 .../model_configs/ViT-B-32-quickgelu.json | 0 .../open_clap/model_configs/ViT-B-32.json | 0 .../open_clap/model_configs/ViT-L-14.json | 0 .../ldm/modules/encoders/open_clap/openai.py | 0 .../modules/encoders/open_clap/pann_model.py | 0 .../modules/encoders/open_clap/pretrained.py | 0 .../modules/encoders/open_clap/timm_model.py | 0 .../modules/encoders/open_clap/tokenizer.py | 0 .../modules/encoders/open_clap/transform.py | 0 .../ldm/modules/encoders/open_clap/utils.py | 0 .../ldm/modules/encoders/open_clap/version.py | 0 .../ldm/modules/image_degradation/__init__.py | 0 .../ldm/modules/image_degradation/bsrgan.py | 0 .../modules/image_degradation/bsrgan_light.py | 0 .../modules/image_degradation/utils/test.png | Bin .../modules/image_degradation/utils_image.py | 0 .../ldm/modules/losses_audio/__init__.py | 0 .../modules/losses_audio/contperceptual.py | 0 .../losses_audio/contperceptual_dis.py | 0 .../ldm/modules/losses_audio/lpaps.py | 0 .../vggishish/config/melception.yaml | 0 .../losses_audio/vggishish/config/vggish.yaml | 0 .../train_means_stds_melspec_10s_22050hz.txt | 0 .../losses_audio/vggishish/data/vggsound.csv | 0 .../vggishish/data/vggsound_test.txt | 0 .../vggishish/data/vggsound_train.txt | 0 .../vggishish/data/vggsound_valid.txt | 0 .../modules/losses_audio/vggishish/dataset.py | 0 .../modules/losses_audio/vggishish/logger.py | 0 .../modules/losses_audio/vggishish/loss.py | 0 .../modules/losses_audio/vggishish/metrics.py | 0 .../modules/losses_audio/vggishish/model.py | 0 .../modules/losses_audio/vggishish/predict.py | 0 .../vggishish/train_melception.py | 0 .../losses_audio/vggishish/train_vggishish.py | 0 .../losses_audio/vggishish/transforms.py | 0 .../ldm/modules/losses_audio/vqperceptual.py | 0 .../ldm/modules/x_transformer.py | 0 .../ldm/util.py | 0 .../useful_ckpts/CLAP/config.yml | 0 .../vocoder/hifigan/modules.py | 0 .../vocoder/logs/hifi_0127/args.yml | 0 .../wav_evaluation/models/CLAPWrapper.py | 0 .../wav_evaluation/models/__init__.py | 0 .../wav_evaluation/models/audio.py | 0 .../wav_evaluation/models/clap.py | 0 .../wav_evaluation/models/utils.py | 0 109 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml create mode 100644 text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml rename text_to_audio/{MakeAnAudio/configs/text-to-audio => Make_An_Audio/configs/text_to_audio}/clap_args.yaml (100%) rename text_to_audio/{MakeAnAudio/configs/text-to-audio => Make_An_Audio/configs/text_to_audio}/hifigan_args.yaml (100%) rename text_to_audio/{MakeAnAudio/configs/text-to-audio => Make_An_Audio/configs/text_to_audio}/txt2audio_args.yaml (100%) create mode 100644 text_to_audio/Make_An_Audio/ldm/data/extract_mel_spectrogram.py rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/lr_scheduler.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/autoencoder.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/autoencoder_multi.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/classifier.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/ddim.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/ddpm.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/ddpm_audio.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/ddpm_audio_inpaint.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/models/diffusion/plms.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/attention.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/diffusionmodules/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/diffusionmodules/custom_openaimodel.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/diffusionmodules/model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/diffusionmodules/openaimodel.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/diffusionmodules/util.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/discriminator/model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/discriminator/multi_window_disc.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/distributions/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/distributions/distributions.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/ema.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/CLAPWrapper.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/audio.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/clap.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/config.yml (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/CLAP/utils.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/modules.py (91%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/bert.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/factory.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/feature_fusion.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/htsat.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/linear_probe.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/loss.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-10.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-14.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/PANN-6.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN101.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN50.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN50x16.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/RN50x4.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/openai.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/pann_model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/pretrained.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/timm_model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/tokenizer.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/transform.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/utils.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/encoders/open_clap/version.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/image_degradation/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/image_degradation/bsrgan.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/image_degradation/bsrgan_light.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/image_degradation/utils/test.png (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/image_degradation/utils_image.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/contperceptual.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/contperceptual_dis.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/lpaps.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/config/melception.yaml (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/config/vggish.yaml (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/data/vggsound.csv (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/data/vggsound_test.txt (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/data/vggsound_train.txt (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/data/vggsound_valid.txt (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/dataset.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/logger.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/loss.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/metrics.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/model.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/predict.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/train_melception.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/train_vggishish.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vggishish/transforms.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/losses_audio/vqperceptual.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/modules/x_transformer.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/ldm/util.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/useful_ckpts/CLAP/config.yml (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/vocoder/hifigan/modules.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/vocoder/logs/hifi_0127/args.yml (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/wav_evaluation/models/CLAPWrapper.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/wav_evaluation/models/__init__.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/wav_evaluation/models/audio.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/wav_evaluation/models/clap.py (100%) rename text_to_audio/{MakeAnAudio => Make_An_Audio}/wav_evaluation/models/utils.py (100%) diff --git a/text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml b/text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml new file mode 100644 index 0000000..917c3de --- /dev/null +++ b/text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml @@ -0,0 +1,77 @@ +model: + base_learning_rate: 1.0e-05 + target: ldm.models.diffusion.ddpm_audio.LatentDiffusion_audio + params: + linear_start: 0.00085 + linear_end: 0.0120 + num_timesteps_cond: 1 + log_every_t: 200 + timesteps: 1000 + first_stage_key: image + cond_stage_key: caption + image_size: 32 # unused + mel_dim: 10 # 80 // 2^3 + mel_length: 78 # 624 // 2^3 + channels: 4 + cond_stage_trainable: false + conditioning_key: crossattn + monitor: val/loss_simple_ema + scale_by_std: True + use_ema: False + + scheduler_config: # 10000 warmup steps + target: ldm.lr_scheduler.LambdaLinearScheduler + params: + warm_up_steps: [10000] + cycle_lengths: [10000000000000] + f_start: [1.e-6] + f_max: [1.] + f_min: [ 1.] + + unet_config: + target: ldm.modules.diffusionmodules.custom_openaimodel.UNetModel + params: + image_size: 32 # ununsed + in_channels: 4 + out_channels: 4 + model_channels: 256 + attention_resolutions: + - 1 + - 2 + num_res_blocks: 2 + channel_mult: # num_down = len(ch_mult)-1 + - 1 + - 2 + num_head_channels: 32 + use_spatial_transformer: true + transformer_depth: 1 + context_dim: 1024 + use_context_project: false + + + first_stage_config: + target: ldm.models.autoencoder.AutoencoderKL + params: + embed_dim: 4 + monitor: val/rec_loss + ddconfig: + double_z: true + z_channels: 4 + resolution: 848 + in_channels: 1 + out_ch: 1 + ch: 128 + ch_mult: [ 1, 2, 2, 4 ] # num_down = len(ch_mult)-1 + num_res_blocks: 2 + attn_resolutions: [106, 212] + dropout: 0.0 + lossconfig: + target: torch.nn.Identity + + cond_stage_config: + target: ldm.modules.encoders.modules.FrozenGlobalNormOpenCLIPEmbedder + params: + freeze: True + delvisual: False + + diff --git a/text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml b/text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml new file mode 100644 index 0000000..7191487 --- /dev/null +++ b/text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml @@ -0,0 +1,68 @@ +model: + base_learning_rate: 1.0e-05 + target: ldm.models.diffusion.ddpm_audio.LatentDiffusion_audio + params: + linear_start: 0.0015 + linear_end: 0.0205 + log_every_t: 100 + timesteps: 1000 + loss_type: l1 + first_stage_key: image + cond_stage_key: masked_image + image_size: 32 # unused + mel_dim: 10 # 80 // 2^3 + mel_length: 106 # 848 // 2^3 + channels: 4 + concat_mode: true + monitor: val/loss + use_ema: False + + scheduler_config: + target: ldm.lr_scheduler.LambdaWarmUpCosineScheduler + params: + verbosity_interval: 0 + warm_up_steps: 1000 + max_decay_steps: 50000 + lr_start: 0.001 + lr_max: 0.1 + lr_min: 0.0001 + + unet_config: + target: ldm.modules.diffusionmodules.openaimodel.UNetModel + params: + image_size: 32 # ununsed + in_channels: 9 # 4 + 1 + 4 + out_channels: 4 + model_channels: 320 + attention_resolutions: + - 1 + - 2 + num_res_blocks: 2 + channel_mult: # num_down = len(ch_mult)-1 + - 1 + - 2 + num_heads: 8 + resblock_updown: true + + first_stage_config: + target: ldm.models.autoencoder.AutoencoderKL + params: + embed_dim: 4 + monitor: val/rec_loss + ckpt_path: # /apdcephfs/share_1316500/nlphuang/results/Text_to_audio/ae15/2022-12-15T22-24-00_mixdata_kl_4_tile/epoch=000009-v2.ckpt + ddconfig: + double_z: true + z_channels: 4 + resolution: 848 + in_channels: 1 + out_ch: 1 + ch: 128 + ch_mult: [ 1, 2, 2, 4 ] # num_down = len(ch_mult)-1 + num_res_blocks: 2 + attn_resolutions: [106, 212] + dropout: 0.0 + lossconfig: + target: torch.nn.Identity + + cond_stage_config: __is_first_stage__ + diff --git a/text_to_audio/MakeAnAudio/configs/text-to-audio/clap_args.yaml b/text_to_audio/Make_An_Audio/configs/text_to_audio/clap_args.yaml similarity index 100% rename from text_to_audio/MakeAnAudio/configs/text-to-audio/clap_args.yaml rename to text_to_audio/Make_An_Audio/configs/text_to_audio/clap_args.yaml diff --git a/text_to_audio/MakeAnAudio/configs/text-to-audio/hifigan_args.yaml b/text_to_audio/Make_An_Audio/configs/text_to_audio/hifigan_args.yaml similarity index 100% rename from text_to_audio/MakeAnAudio/configs/text-to-audio/hifigan_args.yaml rename to text_to_audio/Make_An_Audio/configs/text_to_audio/hifigan_args.yaml diff --git a/text_to_audio/MakeAnAudio/configs/text-to-audio/txt2audio_args.yaml b/text_to_audio/Make_An_Audio/configs/text_to_audio/txt2audio_args.yaml similarity index 100% rename from text_to_audio/MakeAnAudio/configs/text-to-audio/txt2audio_args.yaml rename to text_to_audio/Make_An_Audio/configs/text_to_audio/txt2audio_args.yaml diff --git a/text_to_audio/Make_An_Audio/ldm/data/extract_mel_spectrogram.py b/text_to_audio/Make_An_Audio/ldm/data/extract_mel_spectrogram.py new file mode 100644 index 0000000..384755b --- /dev/null +++ b/text_to_audio/Make_An_Audio/ldm/data/extract_mel_spectrogram.py @@ -0,0 +1,151 @@ +import argparse +import os +import os.path as P +from copy import deepcopy +from functools import partial +from glob import glob +from multiprocessing import Pool +from pathlib import Path + +import librosa +import numpy as np +import torchvision + + +class MelSpectrogram(object): + def __init__(self, sr, nfft, fmin, fmax, nmels, hoplen, spec_power, inverse=False): + self.sr = sr + self.nfft = nfft + self.fmin = fmin + self.fmax = fmax + self.nmels = nmels + self.hoplen = hoplen + self.spec_power = spec_power + self.inverse = inverse + + self.mel_basis = librosa.filters.mel(sr=sr, n_fft=nfft, fmin=fmin, fmax=fmax, n_mels=nmels) + + def __call__(self, x): + if self.inverse: + spec = librosa.feature.inverse.mel_to_stft( + x, sr=self.sr, n_fft=self.nfft, fmin=self.fmin, fmax=self.fmax, power=self.spec_power + ) + wav = librosa.griffinlim(spec, hop_length=self.hoplen) + return wav + else: + spec = np.abs(librosa.stft(x, n_fft=self.nfft, hop_length=self.hoplen)) ** self.spec_power + mel_spec = np.dot(self.mel_basis, spec) + return mel_spec + +class LowerThresh(object): + def __init__(self, min_val, inverse=False): + self.min_val = min_val + self.inverse = inverse + + def __call__(self, x): + if self.inverse: + return x + else: + return np.maximum(self.min_val, x) + +class Add(object): + def __init__(self, val, inverse=False): + self.inverse = inverse + self.val = val + + def __call__(self, x): + if self.inverse: + return x - self.val + else: + return x + self.val + +class Subtract(Add): + def __init__(self, val, inverse=False): + self.inverse = inverse + self.val = val + + def __call__(self, x): + if self.inverse: + return x + self.val + else: + return x - self.val + +class Multiply(object): + def __init__(self, val, inverse=False) -> None: + self.val = val + self.inverse = inverse + + def __call__(self, x): + if self.inverse: + return x / self.val + else: + return x * self.val + +class Divide(Multiply): + def __init__(self, val, inverse=False): + self.inverse = inverse + self.val = val + + def __call__(self, x): + if self.inverse: + return x * self.val + else: + return x / self.val + +class Log10(object): + def __init__(self, inverse=False): + self.inverse = inverse + + def __call__(self, x): + if self.inverse: + return 10 ** x + else: + return np.log10(x) + +class Clip(object): + def __init__(self, min_val, max_val, inverse=False): + self.min_val = min_val + self.max_val = max_val + self.inverse = inverse + + def __call__(self, x): + if self.inverse: + return x + else: + return np.clip(x, self.min_val, self.max_val) + +class TrimSpec(object): + def __init__(self, max_len, inverse=False): + self.max_len = max_len + self.inverse = inverse + + def __call__(self, x): + if self.inverse: + return x + else: + return x[:, :self.max_len] + +class MaxNorm(object): + def __init__(self, inverse=False): + self.inverse = inverse + self.eps = 1e-10 + + def __call__(self, x): + if self.inverse: + return x + else: + return x / (x.max() + self.eps) + + +TRANSFORMS_16000 = torchvision.transforms.Compose([ + MelSpectrogram(sr=16000, nfft=1024, fmin=125, fmax=7600, nmels=80, hoplen=1024//4, spec_power=1), + LowerThresh(1e-5), + Log10(), + Multiply(20), + Subtract(20), + Add(100), + Divide(100), + Clip(0, 1.0) + # TrimSpec(860) +]) + diff --git a/text_to_audio/MakeAnAudio/ldm/lr_scheduler.py b/text_to_audio/Make_An_Audio/ldm/lr_scheduler.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/lr_scheduler.py rename to text_to_audio/Make_An_Audio/ldm/lr_scheduler.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/autoencoder.py b/text_to_audio/Make_An_Audio/ldm/models/autoencoder.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/autoencoder.py rename to text_to_audio/Make_An_Audio/ldm/models/autoencoder.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/autoencoder_multi.py b/text_to_audio/Make_An_Audio/ldm/models/autoencoder_multi.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/autoencoder_multi.py rename to text_to_audio/Make_An_Audio/ldm/models/autoencoder_multi.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/__init__.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/__init__.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/classifier.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/classifier.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/classifier.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/classifier.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/ddim.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/ddim.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/ddim.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/ddim.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm_audio.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm_audio.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm_audio.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm_audio.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm_audio_inpaint.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm_audio_inpaint.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/ddpm_audio_inpaint.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/ddpm_audio_inpaint.py diff --git a/text_to_audio/MakeAnAudio/ldm/models/diffusion/plms.py b/text_to_audio/Make_An_Audio/ldm/models/diffusion/plms.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/models/diffusion/plms.py rename to text_to_audio/Make_An_Audio/ldm/models/diffusion/plms.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/attention.py b/text_to_audio/Make_An_Audio/ldm/modules/attention.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/attention.py rename to text_to_audio/Make_An_Audio/ldm/modules/attention.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/custom_openaimodel.py b/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/custom_openaimodel.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/custom_openaimodel.py rename to text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/custom_openaimodel.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/model.py b/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/model.py rename to text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/openaimodel.py b/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/openaimodel.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/openaimodel.py rename to text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/openaimodel.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/util.py b/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/util.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/diffusionmodules/util.py rename to text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/util.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/discriminator/model.py b/text_to_audio/Make_An_Audio/ldm/modules/discriminator/model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/discriminator/model.py rename to text_to_audio/Make_An_Audio/ldm/modules/discriminator/model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/discriminator/multi_window_disc.py b/text_to_audio/Make_An_Audio/ldm/modules/discriminator/multi_window_disc.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/discriminator/multi_window_disc.py rename to text_to_audio/Make_An_Audio/ldm/modules/discriminator/multi_window_disc.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/distributions/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/distributions/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/distributions/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/distributions/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/distributions/distributions.py b/text_to_audio/Make_An_Audio/ldm/modules/distributions/distributions.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/distributions/distributions.py rename to text_to_audio/Make_An_Audio/ldm/modules/distributions/distributions.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/ema.py b/text_to_audio/Make_An_Audio/ldm/modules/ema.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/ema.py rename to text_to_audio/Make_An_Audio/ldm/modules/ema.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/CLAPWrapper.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/CLAPWrapper.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/CLAPWrapper.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/CLAPWrapper.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/audio.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/audio.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/audio.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/audio.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/clap.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/clap.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/clap.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/clap.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/config.yml b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/config.yml similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/config.yml rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/config.yml diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/utils.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/utils.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/CLAP/utils.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/utils.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/modules.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/modules.py similarity index 91% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/modules.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/modules.py index f6769e9..8a3f8df 100644 --- a/text_to_audio/MakeAnAudio/ldm/modules/encoders/modules.py +++ b/text_to_audio/Make_An_Audio/ldm/modules/encoders/modules.py @@ -9,7 +9,7 @@ from ldm.modules.encoders.CLAP.utils import read_config_as_args from ldm.modules.encoders.CLAP.clap import TextEncoder from ldm.util import default, count_params - +import open_clip class AbstractEncoder(nn.Module): def __init__(self): @@ -310,5 +310,41 @@ def forward(self, text): z = outputs.last_hidden_state return z + def encode(self, text): + return self(text) +class FrozenGlobalNormOpenCLIPEmbedder(AbstractEncoder): + """ + Uses the OpenCLIP transformer encoder for text + """ + def __init__(self, arch="ViT-H-14", version="laion2b_s32b_b79k", device="cuda", freeze=True, delvisual=True): + super().__init__() + model, _, preprocess = open_clip.create_model_and_transforms(arch, device=torch.device('cpu'), pretrained=version) + if delvisual: + del model.visual + del preprocess + else: + self.preprocess = preprocess + self.model = model + + self.device = device + if freeze: + self.freeze() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + tokens = open_clip.tokenize(text) + z = self.model.encode_text(tokens.to(self.device)) + z /= z.norm(dim=-1, keepdim=True) + return z.unsqueeze(1) + + def forward_img(self, image): + z = self.model.encode_image(image.to(self.device)) + z /= z.norm(dim=-1, keepdim=True) + return z.unsqueeze(1) + def encode(self, text): return self(text) \ No newline at end of file diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/bert.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bert.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/bert.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bert.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/factory.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/factory.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/factory.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/factory.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/feature_fusion.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/feature_fusion.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/feature_fusion.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/feature_fusion.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/htsat.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/htsat.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/htsat.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/htsat.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/linear_probe.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/linear_probe.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/linear_probe.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/linear_probe.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/loss.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/loss.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/loss.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/loss.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-10.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-10.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-10.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-10.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-14.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-6.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-6.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/PANN-6.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-6.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN101.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN101.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50x16.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x16.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50x16.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x16.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50x4.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x4.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/RN50x4.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x4.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/openai.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/openai.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/openai.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/openai.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/pann_model.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/pann_model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/pann_model.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/pann_model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/pretrained.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/pretrained.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/pretrained.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/pretrained.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/timm_model.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/timm_model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/timm_model.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/timm_model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/tokenizer.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/tokenizer.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/tokenizer.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/tokenizer.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/transform.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/transform.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/transform.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/transform.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/utils.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/utils.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/utils.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/utils.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/version.py b/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/version.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/encoders/open_clap/version.py rename to text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/version.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/image_degradation/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/image_degradation/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/image_degradation/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/image_degradation/bsrgan.py b/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/bsrgan.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/image_degradation/bsrgan.py rename to text_to_audio/Make_An_Audio/ldm/modules/image_degradation/bsrgan.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/image_degradation/bsrgan_light.py b/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/bsrgan_light.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/image_degradation/bsrgan_light.py rename to text_to_audio/Make_An_Audio/ldm/modules/image_degradation/bsrgan_light.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/image_degradation/utils/test.png b/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils/test.png similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/image_degradation/utils/test.png rename to text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils/test.png diff --git a/text_to_audio/MakeAnAudio/ldm/modules/image_degradation/utils_image.py b/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils_image.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/image_degradation/utils_image.py rename to text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils_image.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/__init__.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/__init__.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/__init__.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/contperceptual.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/contperceptual.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/contperceptual.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/contperceptual.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/contperceptual_dis.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/contperceptual_dis.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/contperceptual_dis.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/contperceptual_dis.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/lpaps.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/lpaps.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/lpaps.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/lpaps.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/config/melception.yaml b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/melception.yaml similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/config/melception.yaml rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/melception.yaml diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/config/vggish.yaml b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/vggish.yaml similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/config/vggish.yaml rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/vggish.yaml diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound.csv b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound.csv similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound.csv rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound.csv diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_test.txt b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_test.txt similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_test.txt rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_test.txt diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_train.txt b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_train.txt similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_train.txt rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_train.txt diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_valid.txt b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_valid.txt similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/data/vggsound_valid.txt rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/vggsound_valid.txt diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/dataset.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/dataset.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/dataset.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/dataset.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/logger.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/logger.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/logger.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/logger.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/loss.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/loss.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/loss.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/loss.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/metrics.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/metrics.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/metrics.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/metrics.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/model.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/model.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/model.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/model.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/predict.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/predict.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/predict.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/predict.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/train_melception.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/train_melception.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/train_melception.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/train_melception.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/train_vggishish.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/train_vggishish.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/train_vggishish.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/train_vggishish.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/transforms.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/transforms.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vggishish/transforms.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/transforms.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vqperceptual.py b/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vqperceptual.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/losses_audio/vqperceptual.py rename to text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vqperceptual.py diff --git a/text_to_audio/MakeAnAudio/ldm/modules/x_transformer.py b/text_to_audio/Make_An_Audio/ldm/modules/x_transformer.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/modules/x_transformer.py rename to text_to_audio/Make_An_Audio/ldm/modules/x_transformer.py diff --git a/text_to_audio/MakeAnAudio/ldm/util.py b/text_to_audio/Make_An_Audio/ldm/util.py similarity index 100% rename from text_to_audio/MakeAnAudio/ldm/util.py rename to text_to_audio/Make_An_Audio/ldm/util.py diff --git a/text_to_audio/MakeAnAudio/useful_ckpts/CLAP/config.yml b/text_to_audio/Make_An_Audio/useful_ckpts/CLAP/config.yml similarity index 100% rename from text_to_audio/MakeAnAudio/useful_ckpts/CLAP/config.yml rename to text_to_audio/Make_An_Audio/useful_ckpts/CLAP/config.yml diff --git a/text_to_audio/MakeAnAudio/vocoder/hifigan/modules.py b/text_to_audio/Make_An_Audio/vocoder/hifigan/modules.py similarity index 100% rename from text_to_audio/MakeAnAudio/vocoder/hifigan/modules.py rename to text_to_audio/Make_An_Audio/vocoder/hifigan/modules.py diff --git a/text_to_audio/MakeAnAudio/vocoder/logs/hifi_0127/args.yml b/text_to_audio/Make_An_Audio/vocoder/logs/hifi_0127/args.yml similarity index 100% rename from text_to_audio/MakeAnAudio/vocoder/logs/hifi_0127/args.yml rename to text_to_audio/Make_An_Audio/vocoder/logs/hifi_0127/args.yml diff --git a/text_to_audio/MakeAnAudio/wav_evaluation/models/CLAPWrapper.py b/text_to_audio/Make_An_Audio/wav_evaluation/models/CLAPWrapper.py similarity index 100% rename from text_to_audio/MakeAnAudio/wav_evaluation/models/CLAPWrapper.py rename to text_to_audio/Make_An_Audio/wav_evaluation/models/CLAPWrapper.py diff --git a/text_to_audio/MakeAnAudio/wav_evaluation/models/__init__.py b/text_to_audio/Make_An_Audio/wav_evaluation/models/__init__.py similarity index 100% rename from text_to_audio/MakeAnAudio/wav_evaluation/models/__init__.py rename to text_to_audio/Make_An_Audio/wav_evaluation/models/__init__.py diff --git a/text_to_audio/MakeAnAudio/wav_evaluation/models/audio.py b/text_to_audio/Make_An_Audio/wav_evaluation/models/audio.py similarity index 100% rename from text_to_audio/MakeAnAudio/wav_evaluation/models/audio.py rename to text_to_audio/Make_An_Audio/wav_evaluation/models/audio.py diff --git a/text_to_audio/MakeAnAudio/wav_evaluation/models/clap.py b/text_to_audio/Make_An_Audio/wav_evaluation/models/clap.py similarity index 100% rename from text_to_audio/MakeAnAudio/wav_evaluation/models/clap.py rename to text_to_audio/Make_An_Audio/wav_evaluation/models/clap.py diff --git a/text_to_audio/MakeAnAudio/wav_evaluation/models/utils.py b/text_to_audio/Make_An_Audio/wav_evaluation/models/utils.py similarity index 100% rename from text_to_audio/MakeAnAudio/wav_evaluation/models/utils.py rename to text_to_audio/Make_An_Audio/wav_evaluation/models/utils.py From 526a05a26818b815a8fa6769bb1433ab9494c624 Mon Sep 17 00:00:00 2001 From: lmz <1352359183@qq.com> Date: Sun, 30 Apr 2023 23:28:28 +0800 Subject: [PATCH 3/4] update --- .../Make_An_Audio/vocoder/bigvgan/__init__.py | 0 .../vocoder/bigvgan/activations.py | 120 +++++ .../bigvgan/alias_free_torch/__init__.py | 6 + .../vocoder/bigvgan/alias_free_torch/act.py | 28 ++ .../bigvgan/alias_free_torch/filter.py | 95 ++++ .../bigvgan/alias_free_torch/resample.py | 49 +++ .../Make_An_Audio/vocoder/bigvgan/models.py | 414 ++++++++++++++++++ 7 files changed, 712 insertions(+) create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/__init__.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/activations.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/__init__.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/act.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/filter.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/resample.py create mode 100644 text_to_audio/Make_An_Audio/vocoder/bigvgan/models.py diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/__init__.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/activations.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/activations.py new file mode 100644 index 0000000..61f2808 --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/activations.py @@ -0,0 +1,120 @@ +# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license. +# LICENSE is in incl_licenses directory. + +import torch +from torch import nn, sin, pow +from torch.nn import Parameter + + +class Snake(nn.Module): + ''' + Implementation of a sine-based periodic activation function + Shape: + - Input: (B, C, T) + - Output: (B, C, T), same shape as the input + Parameters: + - alpha - trainable parameter + References: + - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda: + https://arxiv.org/abs/2006.08195 + Examples: + >>> a1 = snake(256) + >>> x = torch.randn(256) + >>> x = a1(x) + ''' + def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): + ''' + Initialization. + INPUT: + - in_features: shape of the input + - alpha: trainable parameter + alpha is initialized to 1 by default, higher values = higher-frequency. + alpha will be trained along with the rest of your model. + ''' + super(Snake, self).__init__() + self.in_features = in_features + + # initialize alpha + self.alpha_logscale = alpha_logscale + if self.alpha_logscale: # log scale alphas initialized to zeros + self.alpha = Parameter(torch.zeros(in_features) * alpha) + else: # linear scale alphas initialized to ones + self.alpha = Parameter(torch.ones(in_features) * alpha) + + self.alpha.requires_grad = alpha_trainable + + self.no_div_by_zero = 0.000000001 + + def forward(self, x): + ''' + Forward pass of the function. + Applies the function to the input elementwise. + Snake ∶= x + 1/a * sin^2 (xa) + ''' + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] + if self.alpha_logscale: + alpha = torch.exp(alpha) + x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2) + + return x + + +class SnakeBeta(nn.Module): + ''' + A modified Snake function which uses separate parameters for the magnitude of the periodic components + Shape: + - Input: (B, C, T) + - Output: (B, C, T), same shape as the input + Parameters: + - alpha - trainable parameter that controls frequency + - beta - trainable parameter that controls magnitude + References: + - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda: + https://arxiv.org/abs/2006.08195 + Examples: + >>> a1 = snakebeta(256) + >>> x = torch.randn(256) + >>> x = a1(x) + ''' + def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): + ''' + Initialization. + INPUT: + - in_features: shape of the input + - alpha - trainable parameter that controls frequency + - beta - trainable parameter that controls magnitude + alpha is initialized to 1 by default, higher values = higher-frequency. + beta is initialized to 1 by default, higher values = higher-magnitude. + alpha will be trained along with the rest of your model. + ''' + super(SnakeBeta, self).__init__() + self.in_features = in_features + + # initialize alpha + self.alpha_logscale = alpha_logscale + if self.alpha_logscale: # log scale alphas initialized to zeros + self.alpha = Parameter(torch.zeros(in_features) * alpha) + self.beta = Parameter(torch.zeros(in_features) * alpha) + else: # linear scale alphas initialized to ones + self.alpha = Parameter(torch.ones(in_features) * alpha) + self.beta = Parameter(torch.ones(in_features) * alpha) + + self.alpha.requires_grad = alpha_trainable + self.beta.requires_grad = alpha_trainable + + self.no_div_by_zero = 0.000000001 + + def forward(self, x): + ''' + Forward pass of the function. + Applies the function to the input elementwise. + SnakeBeta ∶= x + 1/b * sin^2 (xa) + ''' + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] + beta = self.beta.unsqueeze(0).unsqueeze(-1) + if self.alpha_logscale: + alpha = torch.exp(alpha) + beta = torch.exp(beta) + x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2) + + return x \ No newline at end of file diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/__init__.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/__init__.py new file mode 100644 index 0000000..a2318b6 --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/__init__.py @@ -0,0 +1,6 @@ +# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0 +# LICENSE is in incl_licenses directory. + +from .filter import * +from .resample import * +from .act import * \ No newline at end of file diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/act.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/act.py new file mode 100644 index 0000000..028debd --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/act.py @@ -0,0 +1,28 @@ +# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0 +# LICENSE is in incl_licenses directory. + +import torch.nn as nn +from .resample import UpSample1d, DownSample1d + + +class Activation1d(nn.Module): + def __init__(self, + activation, + up_ratio: int = 2, + down_ratio: int = 2, + up_kernel_size: int = 12, + down_kernel_size: int = 12): + super().__init__() + self.up_ratio = up_ratio + self.down_ratio = down_ratio + self.act = activation + self.upsample = UpSample1d(up_ratio, up_kernel_size) + self.downsample = DownSample1d(down_ratio, down_kernel_size) + + # x: [B,C,T] + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + x = self.downsample(x) + + return x \ No newline at end of file diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/filter.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/filter.py new file mode 100644 index 0000000..7ad6ea8 --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/filter.py @@ -0,0 +1,95 @@ +# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0 +# LICENSE is in incl_licenses directory. + +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +if 'sinc' in dir(torch): + sinc = torch.sinc +else: + # This code is adopted from adefossez's julius.core.sinc under the MIT License + # https://adefossez.github.io/julius/julius/core.html + # LICENSE is in incl_licenses directory. + def sinc(x: torch.Tensor): + """ + Implementation of sinc, i.e. sin(pi * x) / (pi * x) + __Warning__: Different to julius.sinc, the input is multiplied by `pi`! + """ + return torch.where(x == 0, + torch.tensor(1., device=x.device, dtype=x.dtype), + torch.sin(math.pi * x) / math.pi / x) + + +# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License +# https://adefossez.github.io/julius/julius/lowpass.html +# LICENSE is in incl_licenses directory. +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): # return filter [1,1,kernel_size] + even = (kernel_size % 2 == 0) + half_size = kernel_size // 2 + + #For kaiser window + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.: + beta = 0.1102 * (A - 8.7) + elif A >= 21.: + beta = 0.5842 * (A - 21)**0.4 + 0.07886 * (A - 21.) + else: + beta = 0. + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + + # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio + if even: + time = (torch.arange(-half_size, half_size) + 0.5) + else: + time = torch.arange(kernel_size) - half_size + if cutoff == 0: + filter_ = torch.zeros_like(time) + else: + filter_ = 2 * cutoff * window * sinc(2 * cutoff * time) + # Normalize filter to have sum = 1, otherwise we will have a small leakage + # of the constant component in the input signal. + filter_ /= filter_.sum() + filter = filter_.view(1, 1, kernel_size) + + return filter + + +class LowPassFilter1d(nn.Module): + def __init__(self, + cutoff=0.5, + half_width=0.6, + stride: int = 1, + padding: bool = True, + padding_mode: str = 'replicate', + kernel_size: int = 12): + # kernel_size should be even number for stylegan3 setup, + # in this implementation, odd number is also possible. + super().__init__() + if cutoff < -0.: + raise ValueError("Minimum cutoff must be larger than zero.") + if cutoff > 0.5: + raise ValueError("A cutoff above 0.5 does not make sense.") + self.kernel_size = kernel_size + self.even = (kernel_size % 2 == 0) + self.pad_left = kernel_size // 2 - int(self.even) + self.pad_right = kernel_size // 2 + self.stride = stride + self.padding = padding + self.padding_mode = padding_mode + filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size) + self.register_buffer("filter", filter) + + #input [B, C, T] + def forward(self, x): + _, C, _ = x.shape + + if self.padding: + x = F.pad(x, (self.pad_left, self.pad_right), + mode=self.padding_mode) + out = F.conv1d(x, self.filter.expand(C, -1, -1), + stride=self.stride, groups=C) + + return out \ No newline at end of file diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/resample.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/resample.py new file mode 100644 index 0000000..750e6c3 --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/resample.py @@ -0,0 +1,49 @@ +# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0 +# LICENSE is in incl_licenses directory. + +import torch.nn as nn +from torch.nn import functional as F +from .filter import LowPassFilter1d +from .filter import kaiser_sinc_filter1d + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.stride = ratio + self.pad = self.kernel_size // ratio - 1 + self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2 + self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2 + filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, + half_width=0.6 / ratio, + kernel_size=self.kernel_size) + self.register_buffer("filter", filter) + + # x: [B, C, T] + def forward(self, x): + _, C, _ = x.shape + + x = F.pad(x, (self.pad, self.pad), mode='replicate') + x = self.ratio * F.conv_transpose1d( + x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C) + x = x[..., self.pad_left:-self.pad_right] + + return x + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.lowpass = LowPassFilter1d(cutoff=0.5 / ratio, + half_width=0.6 / ratio, + stride=ratio, + kernel_size=self.kernel_size) + + def forward(self, x): + xx = self.lowpass(x) + + return xx \ No newline at end of file diff --git a/text_to_audio/Make_An_Audio/vocoder/bigvgan/models.py b/text_to_audio/Make_An_Audio/vocoder/bigvgan/models.py new file mode 100644 index 0000000..22e8017 --- /dev/null +++ b/text_to_audio/Make_An_Audio/vocoder/bigvgan/models.py @@ -0,0 +1,414 @@ +# Copyright (c) 2022 NVIDIA CORPORATION. +# Licensed under the MIT license. + +# Adapted from https://github.com/jik876/hifi-gan under the MIT license. +# LICENSE is in incl_licenses directory. + + +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.nn import Conv1d, ConvTranspose1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +import numpy as np +from .activations import Snake,SnakeBeta +from .alias_free_torch import * +import os +from omegaconf import OmegaConf + +LRELU_SLOPE = 0.1 + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size*dilation - dilation)/2) + +class AMPBlock1(torch.nn.Module): + def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5), activation=None): + super(AMPBlock1, self).__init__() + self.h = h + + self.convs1 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]))) + ]) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))) + ]) + self.convs2.apply(init_weights) + + self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers + + if activation == 'snake': # periodic nonlinearity with snake function and anti-aliasing + self.activations = nn.ModuleList([ + Activation1d( + activation=Snake(channels, alpha_logscale=h.snake_logscale)) + for _ in range(self.num_layers) + ]) + elif activation == 'snakebeta': # periodic nonlinearity with snakebeta function and anti-aliasing + self.activations = nn.ModuleList([ + Activation1d( + activation=SnakeBeta(channels, alpha_logscale=h.snake_logscale)) + for _ in range(self.num_layers) + ]) + else: + raise NotImplementedError("activation incorrectly specified. check the config file and look for 'activation'.") + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt + x + + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + +class AMPBlock2(torch.nn.Module): + def __init__(self, h, channels, kernel_size=3, dilation=(1, 3), activation=None): + super(AMPBlock2, self).__init__() + self.h = h + + self.convs = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))) + ]) + self.convs.apply(init_weights) + + self.num_layers = len(self.convs) # total number of conv layers + + if activation == 'snake': # periodic nonlinearity with snake function and anti-aliasing + self.activations = nn.ModuleList([ + Activation1d( + activation=Snake(channels, alpha_logscale=h.snake_logscale)) + for _ in range(self.num_layers) + ]) + elif activation == 'snakebeta': # periodic nonlinearity with snakebeta function and anti-aliasing + self.activations = nn.ModuleList([ + Activation1d( + activation=SnakeBeta(channels, alpha_logscale=h.snake_logscale)) + for _ in range(self.num_layers) + ]) + else: + raise NotImplementedError("activation incorrectly specified. check the config file and look for 'activation'.") + + def forward(self, x): + for c, a in zip (self.convs, self.activations): + xt = a(x) + xt = c(xt) + x = xt + x + + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + +class BigVGAN(torch.nn.Module): + # this is our main BigVGAN model. Applies anti-aliased periodic activation for resblocks. + def __init__(self, h): + super(BigVGAN, self).__init__() + self.h = h + + self.num_kernels = len(h.resblock_kernel_sizes) + self.num_upsamples = len(h.upsample_rates) + + # pre conv + self.conv_pre = weight_norm(Conv1d(h.num_mels, h.upsample_initial_channel, 7, 1, padding=3)) + + # define which AMPBlock to use. BigVGAN uses AMPBlock1 as default + resblock = AMPBlock1 if h.resblock == '1' else AMPBlock2 + + # transposed conv-based upsamplers. does not apply anti-aliasing + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)): + self.ups.append(nn.ModuleList([ + weight_norm(ConvTranspose1d(h.upsample_initial_channel // (2 ** i), + h.upsample_initial_channel // (2 ** (i + 1)), + k, u, padding=(k - u) // 2)) + ])) + + # residual blocks using anti-aliased multi-periodicity composition modules (AMP) + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = h.upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)): + self.resblocks.append(resblock(h, ch, k, d, activation=h.activation)) + + # post conv + if h.activation == "snake": # periodic nonlinearity with snake function and anti-aliasing + activation_post = Snake(ch, alpha_logscale=h.snake_logscale) + self.activation_post = Activation1d(activation=activation_post) + elif h.activation == "snakebeta": # periodic nonlinearity with snakebeta function and anti-aliasing + activation_post = SnakeBeta(ch, alpha_logscale=h.snake_logscale) + self.activation_post = Activation1d(activation=activation_post) + else: + raise NotImplementedError("activation incorrectly specified. check the config file and look for 'activation'.") + + self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3)) + + # weight initialization + for i in range(len(self.ups)): + self.ups[i].apply(init_weights) + self.conv_post.apply(init_weights) + + def forward(self, x): + # pre conv + x = self.conv_pre(x) + + for i in range(self.num_upsamples): + # upsampling + for i_up in range(len(self.ups[i])): + x = self.ups[i][i_up](x) + # AMP blocks + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + + # post conv + x = self.activation_post(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + for l_i in l: + remove_weight_norm(l_i) + for l in self.resblocks: + l.remove_weight_norm() + remove_weight_norm(self.conv_pre) + remove_weight_norm(self.conv_post) + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, h, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.d_mult = h.discriminator_channel_mult + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList([ + norm_f(Conv2d(1, int(32*self.d_mult), (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(int(32*self.d_mult), int(128*self.d_mult), (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(int(128*self.d_mult), int(512*self.d_mult), (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(int(512*self.d_mult), int(1024*self.d_mult), (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(int(1024*self.d_mult), int(1024*self.d_mult), (kernel_size, 1), 1, padding=(2, 0))), + ]) + self.conv_post = norm_f(Conv2d(int(1024*self.d_mult), 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, h): + super(MultiPeriodDiscriminator, self).__init__() + self.mpd_reshapes = h.mpd_reshapes + print("mpd_reshapes: {}".format(self.mpd_reshapes)) + discriminators = [DiscriminatorP(h, rs, use_spectral_norm=h.use_spectral_norm) for rs in self.mpd_reshapes] + self.discriminators = nn.ModuleList(discriminators) + + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorR(nn.Module): + def __init__(self, cfg, resolution): + super().__init__() + + self.resolution = resolution + assert len(self.resolution) == 3, \ + "MRD layer requires list with len=3, got {}".format(self.resolution) + self.lrelu_slope = LRELU_SLOPE + + norm_f = weight_norm if cfg.use_spectral_norm == False else spectral_norm + if hasattr(cfg, "mrd_use_spectral_norm"): + print("INFO: overriding MRD use_spectral_norm as {}".format(cfg.mrd_use_spectral_norm)) + norm_f = weight_norm if cfg.mrd_use_spectral_norm == False else spectral_norm + self.d_mult = cfg.discriminator_channel_mult + if hasattr(cfg, "mrd_channel_mult"): + print("INFO: overriding mrd channel multiplier as {}".format(cfg.mrd_channel_mult)) + self.d_mult = cfg.mrd_channel_mult + + self.convs = nn.ModuleList([ + norm_f(nn.Conv2d(1, int(32*self.d_mult), (3, 9), padding=(1, 4))), + norm_f(nn.Conv2d(int(32*self.d_mult), int(32*self.d_mult), (3, 9), stride=(1, 2), padding=(1, 4))), + norm_f(nn.Conv2d(int(32*self.d_mult), int(32*self.d_mult), (3, 9), stride=(1, 2), padding=(1, 4))), + norm_f(nn.Conv2d(int(32*self.d_mult), int(32*self.d_mult), (3, 9), stride=(1, 2), padding=(1, 4))), + norm_f(nn.Conv2d(int(32*self.d_mult), int(32*self.d_mult), (3, 3), padding=(1, 1))), + ]) + self.conv_post = norm_f(nn.Conv2d(int(32 * self.d_mult), 1, (3, 3), padding=(1, 1))) + + def forward(self, x): + fmap = [] + + x = self.spectrogram(x) + x = x.unsqueeze(1) + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, self.lrelu_slope) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + def spectrogram(self, x): + n_fft, hop_length, win_length = self.resolution + x = F.pad(x, (int((n_fft - hop_length) / 2), int((n_fft - hop_length) / 2)), mode='reflect') + x = x.squeeze(1) + x = torch.stft(x, n_fft=n_fft, hop_length=hop_length, win_length=win_length, center=False, return_complex=True) + x = torch.view_as_real(x) # [B, F, TT, 2] + mag = torch.norm(x, p=2, dim =-1) #[B, F, TT] + + return mag + + +class MultiResolutionDiscriminator(nn.Module): + def __init__(self, cfg, debug=False): + super().__init__() + self.resolutions = cfg.resolutions + assert len(self.resolutions) == 3,\ + "MRD requires list of list with len=3, each element having a list with len=3. got {}".\ + format(self.resolutions) + self.discriminators = nn.ModuleList( + [DiscriminatorR(cfg, resolution) for resolution in self.resolutions] + ) + + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(x=y) + y_d_g, fmap_g = d(x=y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +def feature_loss(fmap_r, fmap_g): + loss = 0 + for dr, dg in zip(fmap_r, fmap_g): + for rl, gl in zip(dr, dg): + loss += torch.mean(torch.abs(rl - gl)) + + return loss*2 + + +def discriminator_loss(disc_real_outputs, disc_generated_outputs): + loss = 0 + r_losses = [] + g_losses = [] + for dr, dg in zip(disc_real_outputs, disc_generated_outputs): + r_loss = torch.mean((1-dr)**2) + g_loss = torch.mean(dg**2) + loss += (r_loss + g_loss) + r_losses.append(r_loss.item()) + g_losses.append(g_loss.item()) + + return loss, r_losses, g_losses + + +def generator_loss(disc_outputs): + loss = 0 + gen_losses = [] + for dg in disc_outputs: + l = torch.mean((1-dg)**2) + gen_losses.append(l) + loss += l + + return loss, gen_losses + + + +class VocoderBigVGAN(object): + def __init__(self, ckpt_vocoder,device='cuda'): + vocoder_sd = torch.load(os.path.join(ckpt_vocoder,'best_netG.pt'), map_location='cpu') + + vocoder_args = OmegaConf.load(os.path.join(ckpt_vocoder,'args.yml')) + + self.generator = BigVGAN(vocoder_args) + self.generator.load_state_dict(vocoder_sd['generator']) + self.generator.eval() + + self.device = device + self.generator.to(self.device) + + def vocode(self, spec): + with torch.no_grad(): + if isinstance(spec,np.ndarray): + spec = torch.from_numpy(spec).unsqueeze(0) + spec = spec.to(dtype=torch.float32,device=self.device) + return self.generator(spec).squeeze().cpu().numpy() + + def __call__(self, wav): + return self.vocode(wav) From f61a97c296f66c6d2e3cd0d8d6ed379c8852f5ab Mon Sep 17 00:00:00 2001 From: lmz <1352359183@qq.com> Date: Sun, 30 Apr 2023 23:29:53 +0800 Subject: [PATCH 4/4] update --- audio-chatgpt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/audio-chatgpt.py b/audio-chatgpt.py index faaa4b1..280534b 100644 --- a/audio-chatgpt.py +++ b/audio-chatgpt.py @@ -184,7 +184,7 @@ def txt2audio(self, text, seed = 55, scale = 1.5, ddim_steps = 100, n_samples = def select_best_audio(self, prompt, wav_list): from wav_evaluation.models.CLAPWrapper import CLAPWrapper - clap_model = CLAPWrapper('useful_ckpts/CLAP/CLAP_weights_2022.pth', 'useful_ckpts/CLAP/config.yml', + clap_model = CLAPWrapper('text_to_audio/Make_An_Audio/useful_ckpts/CLAP/CLAP_weights_2022.pth', 'text_to_audio/Make_An_Audio/useful_ckpts/CLAP/config.yml', use_cuda=torch.cuda.is_available()) text_embeddings = clap_model.get_text_embeddings([prompt]) score_list = [] @@ -216,7 +216,7 @@ def __init__(self, device): print("Initializing Make-An-Audio-Image to %s" % device) self.device = device self.sampler = self._initialize_model('text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml', 'text_to_audio/Make_An_Audio/useful_ckpts/ta54_epoch=000216.ckpt', device=device) - self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio_img/vocoder/logs/bigv16k53w',device=device) + self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio/vocoder/logs/bigv16k53w',device=device) def _initialize_model(self, config, ckpt, device): config = OmegaConf.load(config) @@ -420,7 +420,7 @@ def __init__(self, device): print("Initializing Make-An-Audio-inpaint to %s" % device) self.device = device self.sampler = self._initialize_model_inpaint('text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml', 'text_to_audio/Make_An_Audio/useful_ckpts/inpaint7_epoch00047.ckpt') - self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio/vocoder/logs/bigv16k53',device=device) + self.vocoder = VocoderBigVGAN('text_to_audio/Make_An_Audio/vocoder/logs/bigv16k53w',device=device) self.cmap_transform = matplotlib.cm.viridis def _initialize_model_inpaint(self, config, ckpt):