Request: Streaming output

[StuartIanNaylor]

Does it have or is it possible to have a streaming output to reduce sentence latency.

The examples I have seen save complete sentence to file.

I was wondering if a 2 stage buffer could be used where an output pool buffer feeds a callback buffer that is similar to say pyaudio or sounddevice callbacks, where chunks are delivered to create a stream?

[StuartIanNaylor]

I got the fastspeech2/melgan-tfl running Pi400 2.0Ghz TF 2.3.1
.

run time 8.54567575454712 wave duration 11.110748 x 0.769135953
run time 6.87847638130188 wave duration 8.870023 x1 0.77547447

Of the 2 example text inputs using audio_after

run time 9.178293704986572 wave duration 11.110748 x 0.82607343
run time 6.870104551315308 wave duration 8.87002 x 0.774530897

Of the 2 example text inputs using audio_before

I had a problem on run with scikitlearn It seems that scikit-learn has not been built correctly. but export LD_PRELOAD=libgomp.so.1 python3 fixes that.

Streaming that output would be such a great feature as faster than RT on at least a Pi4 @2.0Ghz
Is it because its a 2 stage pass of faststream then melgan and even if 'chunked' the process swap would still cause delay?

You could grammar parse into shorter sentences, but the addition of a streaming output would be so cool :)

Its hard to time really as results do differ here is a sequential run of audio before
8.77754259109497
6.657330274581909

8.832812547683716
6.772733449935913

8.765198469161987
6.640108346939087

9.199273824691772
6.730331182479858

Code used.

import numpy as np
import yaml
import tensorflow as tf
import soundfile as sf
import sys
import time

from tensorflow_tts.processor import LJSpeechProcessor
from tensorflow_tts.processor.ljspeech import valid_symbols

from tensorflow_tts.configs import FastSpeechConfig, FastSpeech2Config
from tensorflow_tts.configs import MelGANGeneratorConfig
from tensorflow_tts.inference import TFAutoModel, AutoConfig, AutoProcessor

from tensorflow_tts.models import TFFastSpeech, TFFastSpeech2
from tensorflow_tts.models import TFMelGANGenerator

print(tf.__version__) # check if >= 2.4.0

# initialize melgan model
with open('examples/melgan/conf/melgan.v1.yaml') as f:
    melgan_config = yaml.load(f, Loader=yaml.Loader)
melgan_config = MelGANGeneratorConfig(**melgan_config["melgan_generator_params"])
melgan = TFMelGANGenerator(config=melgan_config, name='melgan_generator')
melgan._build()
melgan.load_weights("examples/melgan/checkpoints/melgan-1M6.h5")

# initialize FastSpeech model.
with open('examples/fastspeech2/conf/fastspeech2.v1.yaml') as f:
    config = yaml.load(f, Loader=yaml.Loader)
config = FastSpeech2Config(**config["fastspeech2_params"])
fastspeech = TFFastSpeech2(config=config, name="fastspeech2",
                          enable_tflite_convertible=True)
fastspeech._build()
fastspeech.load_weights("examples/fastspeech2/checkpoints/fastspeech2-generator-1500000.h5")

starttime = time.time()
input_text = "Recent research at Harvard has shown meditating\
for as little as 8 weeks, can actually increase the grey matter in the \
parts of the brain responsible for emotional regulation, and learning."

processor = AutoProcessor.from_pretrained(pretrained_path="ljspeech_mapper.json")
input_ids = processor.text_to_sequence(input_text.lower())
input_ids = np.concatenate([input_ids, [len(valid_symbols) - 1]], -1)  # eos.

mel_before, mel_after, duration_outputs, _, _ = fastspeech.inference(
    input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
    speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
    speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
    f0_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
    energy_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
)

audio_before = melgan(mel_before)[0, :, 0]
audio_after = melgan(mel_after)[0, :, 0]

# save to file
sf.write('./audio_before-tf.wav', audio_before, 22050, "PCM_16")
sf.write('./audio_after-tf.wav', audio_after, 22050, "PCM_16")
endtime = time.time()
print(endtime - starttime)

# Concrete Function
fastspeech_concrete_function = fastspeech.inference_tflite.get_concrete_function()

converter = tf.lite.TFLiteConverter.from_concrete_functions(
    [fastspeech_concrete_function]
)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS,
                                       tf.lite.OpsSet.SELECT_TF_OPS]
tflite_model = converter.convert()

# Save the TF Lite model.
with open('fastspeech_quant.tflite', 'wb') as f:
  f.write(tflite_model)

print('Model size is %f MBs.' % (len(tflite_model) / 1024 / 1024.0) )

# Load the TFLite model and allocate tensors.
interpreter = tf.lite.Interpreter(model_path='fastspeech_quant.tflite')

# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

# Prepare input data.
def prepare_input(input_ids):
  input_ids = tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0)
  return (input_ids,
          tf.convert_to_tensor([0], tf.int32),
          tf.convert_to_tensor([1.0], dtype=tf.float32),
          tf.convert_to_tensor([1.0], dtype=tf.float32),
          tf.convert_to_tensor([1.0], dtype=tf.float32))

# Test the model on random input data.
def infer(input_text):
  processor = AutoProcessor.from_pretrained(pretrained_path="ljspeech_mapper.json")
  input_ids = processor.text_to_sequence(input_text.lower())
  interpreter.resize_tensor_input(input_details[0]['index'], 
                                  [1, len(input_ids)])
  interpreter.resize_tensor_input(input_details[1]['index'], 
                                  [1])
  interpreter.resize_tensor_input(input_details[2]['index'], 
                                  [1])
  interpreter.resize_tensor_input(input_details[3]['index'], 
                                  [1])
  interpreter.resize_tensor_input(input_details[4]['index'], 
                                  [1])
  interpreter.allocate_tensors()
  input_data = prepare_input(input_ids)
  for i, detail in enumerate(input_details):
    input_shape = detail['shape_signature']
    interpreter.set_tensor(detail['index'], input_data[i])

  interpreter.invoke()

  # The function `get_tensor()` returns a copy of the tensor data.
  # Use `tensor()` in order to get a pointer to the tensor.
  return (interpreter.get_tensor(output_details[0]['index']),
          interpreter.get_tensor(output_details[1]['index']))
          
starttime = time.time()     
   
input_text = "Recent research at Harvard has shown meditating\
for as little as 8 weeks, can actually increase the grey matter in the \
parts of the brain responsible for emotional regulation, and learning."

decoder_output_tflite, mel_output_tflite = infer(input_text)
#audio_before_tflite = melgan(decoder_output_tflite)[0, :, 0]
audio_after_tflite = melgan(mel_output_tflite)[0, :, 0]

# save to file
#sf.write('./audio_before-tfl1.wav', audio_before_tflite, 22050, "PCM_16")
sf.write('./audio_after-tfl1.wav', audio_after_tflite, 22050, "PCM_16")
endtime = time.time()
print(endtime - starttime)
starttime = time.time()
input_text = "I love TensorFlow Lite converted FastSpeech with quantization. \
The converted model file is of 28.6 Mega bytes."

decoder_output_tflite, mel_output_tflite = infer(input_text)
#audio_before_tflite = melgan(decoder_output_tflite)[0, :, 0]
audio_after_tflite = melgan(mel_output_tflite)[0, :, 0]
#sf.write('./audio_before-tfl2.wav', audio_before_tflite, 22050, "PCM_16")
sf.write('./audio_after-tfl2.wav', audio_after_tflite, 22050, "PCM_16")
endtime = time.time()
print(endtime - starttime)

[maozhiqiang]

how to convert melgan to streaming outputs!

[dathudeptrai]

@manmay-nakhashi

[manmay-nakhashi]

@maozhiqiang @StuartIanNaylor

1. generate mel(fastspech2) till first delimiter.
2. send it to melgan(take a chunk size of 32 or 64) and send it to audio player
3. to make it more faster on edge device you can train your models for 40 mel.(it gives decent output).
4. my model size for fastspeech2 is ~7 mb and melgan is ~1 mb with 40 mel.

[StuartIanNaylor]

Also I have no idea guys but when I was looking I noticed librosa.
I have been playing with https://github.com/google-research/google-research/tree/master/kws_streaming of late and found the tf.signal math libs to be faster than librosa.

Is there any reason why librosa or is it just historical?
There was actually a big improvement.

[manmay-nakhashi]

@dathudeptrai i have this older script can you modify it according to current version and push it for example ?.

import numpy as np
import tensorflow as tf
from tensorflow_tts.processor import LJSpeechProcessor
from tensorflow_tts.processor.ljspeech import symbols, _symbol_to_id

import asyncio
import scipy.io.wavfile
import time
import sys
import math
import pyaudio
punctuations = ['.', '?', '!']
# from multiprocessing import Pool
from multiprocessing.dummy import Pool
split_punctuations = [',', '.', '-', '?', '!', ':', ';']

#fastspeech Get input and output tensors.
interpreter = tf.lite.Interpreter(model_path='tflite_models/fastspeech_quant.tflite')
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
#melgan Get input and output tensors.
mel_interpreter = tf.lite.Interpreter(model_path="tflite_models/mb_melgan_quant.tflite")
mel_input_details = mel_interpreter.get_input_details()
mel_output_details = mel_interpreter.get_output_details()
#initialize text processor
processor = LJSpeechProcessor(None, "english_cleaners")

def generate_sample(ob, preview=True):
    print("* Generating sample...")
    tone_out = ob

    if preview:
        print("* Previewing audio file...")

        bytestream = tone_out.tobytes()
        pya = pyaudio.PyAudio()
        stream = pya.open(format=pya.get_format_from_width(width=4), channels=1, rate=22050, output=True)
        stream.write(bytestream)
        stream.stop_stream()
        stream.close()
        pya.terminate()
        print("* Preview completed!")
        
    else:
        write('sound.wav', SAMPLE_RATE, tone_out)
        print("* Wrote audio file!")


def custom_splitter(text):
  if "{" in text and "}" in text:
    acc = []
    split = text.split("}")
    for word in split:
      if "{" in word:
        acc.append(word + "}")
      else:
        acc.append(word)
    return acc
  else:
    return text.split()

def add_punctuation(text):
  if len(text) < 1:
    return text
  if len(text) < 10:
    if text[-1] in punctuations:
      if text[-1] != ".":
        return text[:-1] + "."
  if text[-1] not in punctuations:
    text += '.'
  return text

def break_chunks(l, n):
  """Yield successive n-sized chunks from l."""
  for i in range(0, len(l), n):
    yield " ".join(l[i:i + n])

def split_by_threshold(text, threshold):
  text_list = text.split()
  
  if len(text_list) <= threshold:
    return [text]

  if threshold < len(text_list) < (threshold*2):
    return list(break_chunks(
        text_list,
        int(math.ceil(len(text_list) / 2))
    ))
  elif (threshold*2) < len(text_list) < (threshold*3):
    return list(break_chunks(
        text_list,
        int(math.ceil(len(text_list) / 3))
    ))
  elif (threshold*3) < len(text_list) < (threshold*4):
    return list(break_chunks(
        text_list,
        int(math.ceil(len(text_list) / 4))
    ))
  else:
    return list(break_chunks(
        text_list,
        int(math.ceil(len(text_list) / 4))
    ))

def synthesize_helper(text, threshold=4):
    text_list = text.split()
    if len(text_list) <= threshold*1.3:
      decoder_output_tflite, mel_output_tflite = infer(text)
      mel_interpreter.resize_tensor_input(mel_input_details[0]['index'], tf.shape(decoder_output_tflite))
      mel_interpreter.allocate_tensors()
      mel_interpreter.set_tensor(mel_input_details[0]['index'],decoder_output_tflite)
      mel_interpreter.invoke()
      audio = mel_interpreter.get_tensor(mel_output_details[0]['index']).squeeze()
      outdir = "results/"
      generate_sample(audio)
      scipy.io.wavfile.write(outdir+input_text+".wav",22050,audio)
      return

    split_by_punc = None
    if len(text_list) >= threshold:
      for punc in split_punctuations:
        if punc in text:
          split_by_punc = text.split(punc)
          break

    chunks = []
    if split_by_punc:
      for sentence in split_by_punc:
        sentence = sentence.strip()
        chunk = split_by_threshold(sentence, threshold)
        chunks += split_by_threshold(sentence, threshold)
    else:
      chunks += split_by_threshold(text, threshold)

    # loop = asyncio.get_event_loop()
    pool = Pool(processes=1)
    combined_wav = np.array([], dtype = 'float32')
    for idx, chunk in enumerate(chunks):
      if len(chunk) > 0:
        text = add_punctuation(chunk)
        decoder_output_tflite, mel_output_tflite = infer(text)
        mel_interpreter.resize_tensor_input(mel_input_details[0]['index'], tf.shape(decoder_output_tflite))
        mel_interpreter.allocate_tensors()
        mel_interpreter.set_tensor(mel_input_details[0]['index'],decoder_output_tflite)
        mel_interpreter.invoke()
        audio = mel_interpreter.get_tensor(mel_output_details[0]['index']).squeeze()
        # loop.run_until_complete(generate_sample(audio)) 
        pool.apply_async(generate_sample, [audio])
        time.sleep(0.9)
        combined_wav = np.concatenate((combined_wav,audio))
    # loop.close()
    outdir = "results/"
    scipy.io.wavfile.write(outdir+input_text+".wav",22050,combined_wav)



# Prepare input data.
def prepare_input(input_ids):
  input_ids = tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0)
  return (input_ids,
          tf.convert_to_tensor([1.0], dtype=tf.float32))

# Test the model
def infer(input_text):
  input_ids = processor.text_to_sequence(input_text.lower())
  interpreter.resize_tensor_input(input_details[0]['index'], 
                                  [1, len(input_ids)])
  interpreter.allocate_tensors()
  input_data = prepare_input(input_ids)
  for i, detail in enumerate(input_details):
    input_shape = detail['shape']
    interpreter.set_tensor(detail['index'], input_data[i])

  interpreter.invoke()

  # The function `get_tensor()` returns a copy of the tensor data.
  # Use `tensor()` in order to get a pointer to the tensor.
  return (interpreter.get_tensor(output_details[0]['index']),
          interpreter.get_tensor(output_details[1]['index']))

for _ in range(1):
  start_time = time.time()
  input_text = sys.argv[1]
  speaker_id = tf.convert_to_tensor([0.0], dtype=tf.float32)
  synthesize_helper(input_text)
  end_time = time.time()

  print(end_time-start_time)

[StuartIanNaylor]

hey guys many thanks its a great TTS :) So good to have such a great sounding offline TTS on something like a Pi

[dathudeptrai]

@manmay-nakhashi sure, let me take a look :D.

[StuartIanNaylor]

PS is there a model trained with n_mel=40?

ValueError: Shapes (384, 40) and (384, 80) are incompatible

[maozhiqiang]

hi @manmay-nakhashi @dathudeptrai ！ thanks for your reply, My point is, How to use melgan to streaming generation for real mels ? There will be a problem of breakpoint when use frame synthesis ! how to solve this？

[maozhiqiang]

PS is there a model trained with n_mel=40?

ValueError: Shapes (384, 40) and (384, 80) are incompatible

you can modify config file to 40 bin, and Re extracting audio features

[StuartIanNaylor]

I would have to retrain a model though wouldn't I as current one is with 80 n_mel.
Just wondering if there is a demo model completed with 40 n_mel as my 1050ti might not be up for the job :)

PS dunno if any interest but this is the tensorflow audio_ops.audio_spectrogram vs librosa

import tensorflow.compat.v1 as tf
from tensorflow.python.ops import gen_audio_ops as audio_ops
import sounddevice as sd
import numpy as np
import threading
import os

rec_duration = 0.020
sample_rate = 16000
num_channels = 1

def get_mfcc(waveform):
        # Run the spectrogram and MFCC ops to get a 2D audio: Short-time FFTs
        # background_clamp dims: [time, channels]
        sample_rate = 16000

        spectrogram = audio_ops.audio_spectrogram(
            waveform,
            window_size=320,
            stride=160)
        # spectrogram: [channels/batch, frames, fft_feature]

        # extract mfcc features from spectrogram by audio_ops.mfcc:
        # 1 Input is spectrogram frames.
        # 2 Weighted spectrogram into bands using a triangular mel filterbank
        # 3 Logarithmic scaling
        # 4 Discrete cosine transform (DCT), return lowest dct_coefficient_count
        mfccs = audio_ops.mfcc(
            spectrogram=spectrogram,
            sample_rate=sample_rate,
            upper_frequency_limit=7600,
            lower_frequency_limit=60,
            filterbank_channel_count=40,
            dct_coefficient_count=13)
        # mfcc: [channels/batch, frames, dct_coefficient_count]
        # remove channel dim
        mfccs = tf.squeeze(mfccs, axis=0)
        return mfccs



def sd_callback(rec, frames, time, status):
    
    # Notify if errors
    if status:
        print('Error:', status)
    mfcc = get_mfcc(rec)
    #print(mfcc)
    print(os.getloadavg())

# Start streaming from microphone
with sd.InputStream(channels=num_channels,
                    samplerate=sample_rate,
                    blocksize=int(sample_rate * rec_duration),
                    callback=sd_callback):
    threading.Event().wait()

Gives an av of (0.2, 0.2, 0.38)
But with librosa
(0.3, 0.29, 0.27)
Guess not huge but the TF math does seem to be faster

import sounddevice as sd
import numpy as np
import librosa
import threading
import os

rec_duration = 0.020
sample_rate = 16000
num_channels = 1
sd.default.never_drop_input= False
sd.default.latency= ('high', 'high')
sd.default.dtype= ('float32', 'float32')



def get_mfcc(audio):
    audio = np.squeeze(audio, 1)    
    mfccs = librosa.feature.mfcc(y=audio, n_mfcc=13, sr=sample_rate, n_fft=320, hop_length=160, n_mels=40, fmin=60.0, fmax=76000.0, htk=False )
    return mfccs

def sd_callback(rec, frames, time, status):

    # Notify if errors
    if status:
        print('Error:', status)
    mfcc = get_mfcc(rec)
    #print(mfcc)
    print(os.getloadavg())
# Start streaming from microphone
with sd.InputStream(channels=num_channels,
                    samplerate=sample_rate,
                    blocksize=int(sample_rate * rec_duration),
                    callback=sd_callback):
    threading.Event().wait()
    ```
    
    That is with MFCC but thinks generally its the same.