Skip to content

Commit

Permalink
infinite generation with text chords / continue
Browse files Browse the repository at this point in the history
  • Loading branch information
sakemin committed Oct 12, 2023
1 parent 7a2cab8 commit c3758d7
Show file tree
Hide file tree
Showing 112 changed files with 277 additions and 739 deletions.
Binary file added __pycache__/predict.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/__pycache__/environment.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/adversarial/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/adversarial/__pycache__/losses.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/audio.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/audio_dataset.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/audio_utils.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/info_audio_dataset.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/music_dataset.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/sound_dataset.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/data/__pycache__/zip.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/losses/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/losses/__pycache__/balancer.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/losses/__pycache__/sisnr.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/losses/__pycache__/specloss.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/losses/__pycache__/stftloss.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/chroma_cosinesim.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/clap_consistency.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/fad.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/kld.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/rvm.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/metrics/__pycache__/visqol.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/audiogen.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/builders.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/encodec.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/lm.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/loaders.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/multibanddiffusion.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/musicgen.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/models/__pycache__/unet.cpython-39.pyc
Binary file not shown.
484 changes: 0 additions & 484 deletions audiocraft/models/musicgen copy.py

This file was deleted.

91 changes: 91 additions & 0 deletions audiocraft/models/musicgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,30 @@ def generate_continuation(self, prompt: torch.Tensor, prompt_sample_rate: int,
if return_tokens:
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_continuation_with_audio_token(self, prompt,
descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None,
progress: bool = False, return_tokens: bool = False) \
-> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
"""Generate samples conditioned on audio prompts.
Args:
prompt (torch.Tensor): A batch of waveforms used for continuation.
Prompt should be [B, C, T], or [C, T] if only one sample is generated.
prompt_sample_rate (int): Sampling rate of the given audio waveforms.
descriptions (list of str, optional): A list of strings used as text conditioning. Defaults to None.
progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
"""

if descriptions is None:
descriptions = [None] * len(prompt)
attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, None)
assert prompt_tokens is None
prompt_tokens = prompt
tokens = self._generate_tokens(attributes, prompt_tokens, progress)
if return_tokens:
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_continuation_with_audio_chroma(self, prompt: torch.Tensor, prompt_sample_rate: int, melody_wavs: MelodyType,
melody_sample_rate: int, descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None,
Expand Down Expand Up @@ -298,6 +322,46 @@ def generate_continuation_with_audio_chroma(self, prompt: torch.Tensor, prompt_s
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_continuation_with_audio_tokens_and_audio_chroma(self, prompt, melody_wavs: MelodyType,
melody_sample_rate: int, descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None,
progress: bool = False, return_tokens: bool = False) \
-> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
"""Generate samples conditioned on audio prompts.
Args:
prompt (torch.Tensor): A batch of waveforms used for continuation.
Prompt should be [B, C, T], or [C, T] if only one sample is generated.
prompt_sample_rate (int): Sampling rate of the given audio waveforms.
descriptions (list of str, optional): A list of strings used as text conditioning. Defaults to None.
progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
"""
if isinstance(melody_wavs, torch.Tensor):
if melody_wavs.dim() == 2:
melody_wavs = melody_wavs[None]
if melody_wavs.dim() != 3:
raise ValueError("Melody wavs should have a shape [B, C, T].")
melody_wavs = list(melody_wavs)
else:
for melody in melody_wavs:
if melody is not None:
assert melody.dim() == 2, "One melody in the list has the wrong number of dims."

melody_wavs = [
convert_audio(wav, melody_sample_rate, self.sample_rate, self.audio_channels)
if wav is not None else None
for wav in melody_wavs]

if descriptions is None:
descriptions = [None] * len(prompt)

attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None, melody_wavs=melody_wavs)
assert prompt_tokens is None
prompt_tokens = prompt
tokens = self._generate_tokens(attributes, prompt_tokens, progress)
if return_tokens:
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_continuation_with_text_chroma(self, prompt: torch.Tensor, prompt_sample_rate: int, descriptions: tp.List[str], chord_texts: tp.Union[tp.List[str],str],
progress: bool = False, bpm: tp.Union[float,int,tp.List[float],tp.List[int]] = 120, meter: tp.Optional[tp.Union[int,tp.List[int]]] = 4,
return_tokens: bool = False) -> tp.Union[torch.Tensor,
Expand Down Expand Up @@ -329,6 +393,33 @@ def generate_continuation_with_text_chroma(self, prompt: torch.Tensor, prompt_sa
if return_tokens:
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_continuation_with_audio_tokens_and_text_chroma(self, prompt, descriptions: tp.List[str], chord_texts: tp.Union[tp.List[str],str],
progress: bool = False, bpm: tp.Union[float,int,tp.List[float],tp.List[int]] = 120, meter: tp.Optional[tp.Union[int,tp.List[int]]] = 4,
return_tokens: bool = False) -> tp.Union[torch.Tensor,
tp.Tuple[torch.Tensor, torch.Tensor]]:
"""Generate samples conditioned on text and melody.
Args:
descriptions (list of str): A list of strings used as text conditioning.
melody_wavs: (torch.Tensor or list of Tensor): A batch of waveforms used as
melody conditioning. Should have shape [B, C, T] with B matching the description length,
C=1 or 2. It can be [C, T] if there is a single description. It can also be
a list of [C, T] tensors.
melody_sample_rate: (int): Sample rate of the melody waveforms.
progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
"""

if isinstance(chord_texts, str):
chord_texts = [chord_texts]

attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None,
melody_wavs=chord_texts, bpm=bpm, meter=meter)
prompt_tokens = prompt
tokens = self._generate_tokens(attributes, prompt_tokens, progress)
if return_tokens:
return self.generate_audio(tokens), tokens
return self.generate_audio(tokens)

def generate_with_text_chroma(self, descriptions: tp.List[str], chord_texts: tp.Union[tp.List[str],str],
progress: bool = False, bpm: tp.Union[float,int,tp.List[float],tp.List[int]] = 120, meter: tp.Optional[tp.Union[int,tp.List[int]]] = 4,
Expand Down
Binary file modified audiocraft/modules/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/activations.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/chord_chroma.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/chroma.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/codebooks_patterns.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/conditioners.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/conv.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/diffusion_schedule.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/lstm.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/rope.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/seanet.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/streaming.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/__pycache__/transformer.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/__pycache__/btc_model.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/beat.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/chord.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/io.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/key.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/onset.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/tempo.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/modules/btc/mir_eval/__pycache__/util.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/utils/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/utils/__pycache__/chords.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/modules/btc/utils/__pycache__/hparams.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
9 changes: 8 additions & 1 deletion audiocraft/modules/conditioners.py
Original file line number Diff line number Diff line change
Expand Up @@ -757,7 +757,8 @@ def __init__(self, output_dim: int, sample_rate: int, n_chroma: int, radix2_exp:

self.chroma = ChordExtractor(device = device, sample_rate=sample_rate, n_chroma=n_chroma, max_duration = duration, chroma_len = self.chroma_len, winhop = self.winhop).to(device)
self.chords = chords.Chords()


self.continuation_count = 0 # for infinite generation with text chroma
#3 Layered MLP projection override
'''
self.output_proj = nn.Sequential(
Expand Down Expand Up @@ -871,6 +872,9 @@ def _extract_chroma_chunk(self, full_chroma: torch.Tensor, x: WavCondition, idx:
out = F.pad(out[None], (0, 0, 0, target_length - out.shape[0]))[0]
return out.to(self.device)

def set_continuation_count(self, sub_duration_ratio, current_iter):
self.continuation_count = int(self.chroma_len * sub_duration_ratio * current_iter)

@torch.no_grad()
def _get_wav_embedding(self, x: tp.Union[WavCondition, WavChordTextCondition]) -> torch.Tensor:
"""Get the wav embedding from the WavCondition.
Expand Down Expand Up @@ -920,6 +924,9 @@ def _get_wav_embedding(self, x: tp.Union[WavCondition, WavChordTextCondition]) -
mhot = self.chords.chord(token)
rolled = np.roll(mhot[2], mhot[0])
for i in range(count, count + add_step):
if self.continuation_count > 0:
self.continuation_count -= 1
continue
if count >= self.chroma_len:
break
chroma[i] = torch.Tensor(rolled)
Expand Down
Binary file modified audiocraft/optim/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/optim/__pycache__/cosine_lr_scheduler.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/optim/__pycache__/dadam.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/optim/__pycache__/ema.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/optim/__pycache__/fsdp.cpython-39.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified audiocraft/quantization/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/quantization/__pycache__/base.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/quantization/__pycache__/core_vq.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/quantization/__pycache__/vq.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/audiogen.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/base.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/builders.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/compression.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/diffusion.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/solvers/__pycache__/musicgen.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/autocast.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/best_state.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/cache.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/checkpoint.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/cluster.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/deadlock.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/profiler.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/__pycache__/utils.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/samples/__pycache__/__init__.cpython-39.pyc
Binary file not shown.
Binary file modified audiocraft/utils/samples/__pycache__/manager.cpython-39.pyc
Binary file not shown.
79 changes: 0 additions & 79 deletions model_cards/AUDIOGEN_MODEL_CARD.md

This file was deleted.

0 comments on commit c3758d7

Please sign in to comment.