Average Loss stays at 4.5 #1025

sogoodbinar · 2021-12-19T19:27:32Z

sogoodbinar
Dec 19, 2021

Thanks for making this project, makes the whole TTS process so much easier. During fine-tuning a tacotron 2 DDC model, however, the loss stays at 4.5 after around 200 Epochs. I have 500 voice samples (around 40 minutes), which has a good variety. It's a custom dataset, which was modeled after the LJSpeech set. Here is the config, tensorboard and dataset. I've also played around with glow TTS and VITS, but I encounter the same issue. Do I just need more data?

Config:
{
    "model": "tacotron2",
    "run_name": "coqui_tts",
    "run_description": "",
    "epochs": 1000,
    "batch_size": 64,
    "eval_batch_size": 16,
    "mixed_precision": false,
    "scheduler_after_epoch": false,
    "run_eval": true,
    "test_delay_epochs": -1,
    "print_eval": true,
    "dashboard_logger": "tensorboard",
    "print_step": 25,
    "plot_step": 100,
    "model_param_stats": false,
    "project_name": null,
    "log_model_step": null,
    "wandb_entity": null,
    "save_step": 10000,
    "checkpoint": true,
    "keep_all_best": false,
    "keep_after": 10000,
    "num_loader_workers": 4,
    "num_eval_loader_workers": 4,
    "use_noise_augment": false,
    "output_path": "/home/rio/Desktop/TTS/Coqui/config",
    "distributed_backend": "nccl",
    "distributed_url": "tcp://localhost:54321",
    "audio": {
        "fft_size": 1024,
        "win_length": 1024,
        "hop_length": 256,
        "frame_shift_ms": null,
        "frame_length_ms": null,
        "stft_pad_mode": "reflect",
        "sample_rate": 22050,
        "resample": false,
        "preemphasis": 0.0,
        "ref_level_db": 20,
        "do_sound_norm": false,
        "log_func": "np.log",
        "do_trim_silence": false,
        "trim_db": 60.0,
        "power": 1.5,
        "griffin_lim_iters": 60,
        "num_mels": 80,
        "mel_fmin": 0.0,
        "mel_fmax": 8000,
        "spec_gain": 1.0,
        "do_amp_to_db_linear": true,
        "do_amp_to_db_mel": true,
        "signal_norm": false,
        "min_level_db": -100,
        "symmetric_norm": true,
        "max_norm": 4.0,
        "clip_norm": true,
        "stats_path": null
    },
    "use_phonemes": true,
    "use_espeak_phonemes": true,
    "phoneme_language": "en-us",
    "compute_input_seq_cache": false,
    "text_cleaner": "phoneme_cleaners",
    "enable_eos_bos_chars": false,
    "test_sentences_file": "",
    "phoneme_cache_path": "/home/rio/Desktop/TTS/Coqui/config/phoneme_cache",
    "characters": {
        "pad": "_",
        "eos": "~",
        "bos": "^",
        "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;? ",
        "punctuations": "!'(),-.:;? ",
        "phonemes": "iy\u0268\u0289\u026fu\u026a\u028f\u028ae\u00f8\u0258\u0259\u0275\u0264o\u025b\u0153\u025c\u025e\u028c\u0254\u00e6\u0250a\u0276\u0251\u0252\u1d7b\u0298\u0253\u01c0\u0257\u01c3\u0284\u01c2\u0260\u01c1\u029bpbtd\u0288\u0256c\u025fk\u0261q\u0262\u0294\u0274\u014b\u0272\u0273n\u0271m\u0299r\u0280\u2c71\u027e\u027d\u0278\u03b2fv\u03b8\u00f0sz\u0283\u0292\u0282\u0290\u00e7\u029dx\u0263\u03c7\u0281\u0127\u0295h\u0266\u026c\u026e\u028b\u0279\u027bj\u0270l\u026d\u028e\u029f\u02c8\u02cc\u02d0\u02d1\u028dw\u0265\u029c\u02a2\u02a1\u0255\u0291\u027a\u0267\u02b2\u025a\u02de\u026b",
        "unique": true
    },
    "batch_group_size": 0,
    "loss_masking": true,
    "sort_by_audio_len": false,
    "min_seq_len": 1,
    "max_seq_len": Infinity,
    "compute_f0": false,
    "compute_linear_spec": false,
    "add_blank": false,
    "datasets": [
        {
            "name": "ljspeech",
            "path": "/home/rio/Desktop/TTS/Coqui/Datasets/rio",
            "meta_file_train": "metadata.csv",
            "ununsed_speakers": null,
            "meta_file_val": "",
            "meta_file_attn_mask": ""
        }
    ],
    "optimizer": "RAdam",
    "optimizer_params": {
        "betas": [
            0.9,
            0.998
        ],
        "weight_decay": 1e-06
    },
    "lr_scheduler": "NoamLR",
    "lr_scheduler_params": {
        "warmup_steps": 4000
    },
    "test_sentences": [
        "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
        "Be a voice, not an echo.",
        "I'm sorry Dave. I'm afraid I can't do that.",
        "This cake is great. It's so delicious and moist.",
        "Prior to November 22, 1963."
    ],
    "use_gst": false,
    "gst": null,
    "gst_style_input": null,
    "num_speakers": 1,
    "num_chars": 130,
    "r": 6,
    "gradual_training": [
        [
            0,
            6,
            64
        ],
        [
            10000,
            4,
            32
        ],
        [
            50000,
            3,
            32
        ],
        [
            100000,
            2,
            32
        ]
    ],
    "memory_size": -1,
    "prenet_type": "original",
    "prenet_dropout": true,
    "prenet_dropout_at_inference": false,
    "stopnet": true,
    "separate_stopnet": true,
    "stopnet_pos_weight": 10.0,
    "max_decoder_steps": 500,
    "encoder_in_features": 512,
    "decoder_in_features": 512,
    "decoder_output_dim": 80,
    "out_channels": 80,
    "attention_type": "original",
    "attention_heads": null,
    "attention_norm": "sigmoid",
    "attention_win": false,
    "windowing": false,
    "use_forward_attn": false,
    "forward_attn_mask": false,
    "transition_agent": false,
    "location_attn": true,
    "bidirectional_decoder": false,
    "double_decoder_consistency": true,
    "ddc_r": 6,
    "use_speaker_embedding": false,
    "speaker_embedding_dim": 512,
    "use_d_vector_file": false,
    "d_vector_file": false,
    "d_vector_dim": null,
    "lr": 0.0001,
    "grad_clip": 5.0,
    "seq_len_norm": false,
    "decoder_loss_alpha": 0.25,
    "postnet_loss_alpha": 0.25,
    "postnet_diff_spec_alpha": 0.25,
    "decoder_diff_spec_alpha": 0.25,
    "decoder_ssim_alpha": 0.25,
    "postnet_ssim_alpha": 0.25,
    "ga_alpha": 5.0
}

Tensorflow:

Drive link to dataset:
https://drive.google.com/drive/folders/17OfI6xk1kyvZspGYf1_Aendl-7p-haNX?usp=sharing