tutorials/music-with-ml/1.intro-to-magenta.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Music with ML\n",
    "\n",
    "Use magenta to make music with ML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install requirements \n",
    "\n",
    "See this tutorial's workflow file for required system installations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -qU setuptools\n",
    "!pip install -qU pyfluidsynth pretty_midi\n",
    "!pip install -qU magenta\n",
    "!pip install -qU gsutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import magenta\n",
    "import note_seq\n",
    "import tensorflow\n",
    "\n",
    "print(magenta.__version__)\n",
    "print(tensorflow.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Making sounds with NoteSequences\n",
    "\n",
    "Everything in Magenta is centered around NoteSequences. This is an abstract representation of a series of notes, each with different pitches, instruments and strike velocities, much like MIDI.\n",
    "\n",
    "For example, this is a NoteSequence that represents \"Twinkle Twinkle Little Star\". Try changing the pitches to see how the sound changes!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from note_seq.protobuf import music_pb2\n",
    "\n",
    "twinkle_twinkle = music_pb2.NoteSequence()\n",
    "\n",
    "# Add the notes to the sequence.\n",
    "twinkle_twinkle.notes.add(pitch=60, start_time=0.0, end_time=0.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=60, start_time=0.5, end_time=1.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=67, start_time=1.0, end_time=1.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=67, start_time=1.5, end_time=2.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=69, start_time=2.0, end_time=2.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=69, start_time=2.5, end_time=3.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=67, start_time=3.0, end_time=4.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=65, start_time=4.0, end_time=4.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=65, start_time=4.5, end_time=5.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=64, start_time=5.0, end_time=5.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=64, start_time=5.5, end_time=6.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=62, start_time=6.0, end_time=6.5, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=62, start_time=6.5, end_time=7.0, velocity=80)\n",
    "twinkle_twinkle.notes.add(pitch=60, start_time=7.0, end_time=8.0, velocity=80)\n",
    "twinkle_twinkle.total_time = 8\n",
    "\n",
    "twinkle_twinkle.tempos.add(qpm=60)\n",
    "\n",
    "# This is a utility method that visualizes a NoteSequence.\n",
    "note_seq.plot_sequence(twinkle_twinkle)\n",
    "\n",
    "# This is a utility method that plays a NoteSequence.\n",
    "note_seq.play_sequence(twinkle_twinkle, synth=note_seq.fluidsynth)\n",
    "\n",
    "# Here's another NoteSequence!\n",
    "teapot = music_pb2.NoteSequence()\n",
    "teapot.notes.add(pitch=69, start_time=0, end_time=0.5, velocity=80)\n",
    "teapot.notes.add(pitch=71, start_time=0.5, end_time=1, velocity=80)\n",
    "teapot.notes.add(pitch=73, start_time=1, end_time=1.5, velocity=80)\n",
    "teapot.notes.add(pitch=74, start_time=1.5, end_time=2, velocity=80)\n",
    "teapot.notes.add(pitch=76, start_time=2, end_time=2.5, velocity=80)\n",
    "teapot.notes.add(pitch=81, start_time=3, end_time=4, velocity=80)\n",
    "teapot.notes.add(pitch=78, start_time=4, end_time=5, velocity=80)\n",
    "teapot.notes.add(pitch=81, start_time=5, end_time=6, velocity=80)\n",
    "teapot.notes.add(pitch=76, start_time=6, end_time=8, velocity=80)\n",
    "teapot.total_time = 8\n",
    "\n",
    "teapot.tempos.add(qpm=60)\n",
    "\n",
    "note_seq.plot_sequence(teapot)\n",
    "note_seq.play_sequence(teapot, synth=note_seq.synthesize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can use other instruments for your sequences. Drums:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drums = music_pb2.NoteSequence()\n",
    "\n",
    "drums.notes.add(\n",
    "    pitch=36,\n",
    "    start_time=0,\n",
    "    end_time=0.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=38,\n",
    "    start_time=0,\n",
    "    end_time=0.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=0,\n",
    "    end_time=0.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=46,\n",
    "    start_time=0,\n",
    "    end_time=0.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=0.25,\n",
    "    end_time=0.375,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=0.375,\n",
    "    end_time=0.5,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=0.5,\n",
    "    end_time=0.625,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=50,\n",
    "    start_time=0.5,\n",
    "    end_time=0.625,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=36,\n",
    "    start_time=0.75,\n",
    "    end_time=0.875,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=38,\n",
    "    start_time=0.75,\n",
    "    end_time=0.875,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=0.75,\n",
    "    end_time=0.875,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=45,\n",
    "    start_time=0.75,\n",
    "    end_time=0.875,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=36,\n",
    "    start_time=1,\n",
    "    end_time=1.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=1,\n",
    "    end_time=1.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=46,\n",
    "    start_time=1,\n",
    "    end_time=1.125,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=42,\n",
    "    start_time=1.25,\n",
    "    end_time=1.375,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=48,\n",
    "    start_time=1.25,\n",
    "    end_time=1.375,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.notes.add(\n",
    "    pitch=50,\n",
    "    start_time=1.25,\n",
    "    end_time=1.375,\n",
    "    is_drum=True,\n",
    "    instrument=10,\n",
    "    velocity=80,\n",
    ")\n",
    "drums.total_time = 1.375\n",
    "\n",
    "drums.tempos.add(qpm=60)\n",
    "\n",
    "# This is a utility method that visualizes a NoteSequence.\n",
    "note_seq.plot_sequence(drums)\n",
    "\n",
    "# This is a utility method that plays a NoteSequence.\n",
    "note_seq.play_sequence(drums, synth=note_seq.fluidsynth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Using Machine Learning to make music\n",
    "\n",
    "note_seq has several Machine Learning models, each with different strengths. All models are built with Tensorflow, so they will run faster if you can run them on a GPU. Here are some of the most popular ones:\n",
    "\n",
    "MelodyRNN - you give it a NoteSequence, and it continues it in the style of your original NoteSequence.\n",
    "MusicVAE - generates brand new NoteSequences or interpolates between two sequences.\n",
    "Onsets and Frames -- transcribes piano audio\n",
    "Now that we know how to use NoteSequences, adding some basic Machine Learning is a continuation of that. The pattern for using any of these models is:\n",
    "\n",
    "Load note_seq (which we already know how to do!)\n",
    "Create a model from a downloaded checkpoint (i.e. where the weights, or the encoding, of the model lives)\n",
    "Ask the model to do something.\n",
    "Melody RNN\n",
    "A MelodyRNN is an LSTM-based language model for musical notes -- it is best at continuing a NoteSequence that you give it.\n",
    "\n",
    "To use it, you need to give it a sequence to continue and the model will return the following sequence.\n",
    "\n",
    "This example shows how to use the basic Melody RNN model -- check out the docs for the other models, such as lookback_rnn and attention_rnn."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Downloading model bundle. This will take less than a minute...\")\n",
    "note_seq.notebook_utils.download_bundle(\"basic_rnn.mag\", \".\")\n",
    "\n",
    "# Import dependencies.\n",
    "from magenta.models.melody_rnn import melody_rnn_sequence_generator\n",
    "from magenta.models.shared import sequence_generator_bundle\n",
    "from note_seq.protobuf import generator_pb2\n",
    "from note_seq.protobuf import music_pb2\n",
    "\n",
    "# Initialize the model.\n",
    "print(\"Initializing Melody RNN...\")\n",
    "bundle = sequence_generator_bundle.read_bundle_file(\"./basic_rnn.mag\")\n",
    "generator_map = melody_rnn_sequence_generator.get_generator_map()\n",
    "melody_rnn = generator_map[\"basic_rnn\"](checkpoint=None, bundle=bundle)\n",
    "melody_rnn.initialize()\n",
    "\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Continuing a sequence\n",
    "\n",
    "With Melody RNN, you can configure the number of steps the new sequence will be, as well as the \"temperature\" of the result -- the higher the temperature, the more random (and less like the input) your sequence will be. You can play around with these values and see how the resulting sequences are different:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model options. Change these to get different generated sequences!\n",
    "\n",
    "input_sequence = twinkle_twinkle  # change this to teapot if you want\n",
    "num_steps = 128  # change this for shorter or longer sequences\n",
    "temperature = 1.0  # the higher the temperature the more random the sequence.\n",
    "\n",
    "# Set the start time to begin on the next step after the last note ends.\n",
    "last_end_time = (\n",
    "    max(n.end_time for n in input_sequence.notes)\n",
    "    if input_sequence.notes\n",
    "    else 0\n",
    ")\n",
    "qpm = input_sequence.tempos[0].qpm\n",
    "seconds_per_step = 60.0 / qpm / melody_rnn.steps_per_quarter\n",
    "total_seconds = num_steps * seconds_per_step\n",
    "\n",
    "generator_options = generator_pb2.GeneratorOptions()\n",
    "generator_options.args[\"temperature\"].float_value = temperature\n",
    "generate_section = generator_options.generate_sections.add(\n",
    "    start_time=last_end_time + seconds_per_step, end_time=total_seconds\n",
    ")\n",
    "\n",
    "# Ask the model to continue the sequence.\n",
    "sequence = melody_rnn.generate(input_sequence, generator_options)\n",
    "\n",
    "note_seq.plot_sequence(sequence)\n",
    "note_seq.play_sequence(sequence, synth=note_seq.fluidsynth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Music VAE\n",
    "\n",
    "A MusicVAE is a variational autoencoder made up of an Encoder and Decoder -- you can think of the encoder as trying to summarize all the data you give it, and the decoder as trying to recreate the original data, based on this summarized version. As a generative model, you can think of a VAE as coming up with new sequences that could be a decoding of some summarized version.\n",
    "\n",
    "The Music VAE implementation in magenta/music in particular does two things: it can create new sequences (which are reconstructions or variations of the input data), or it can interpolate between two."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Copying checkpoint from GCS. This will take less than a minute...\")\n",
    "# This will download the mel_2bar_big checkpoint. There are more checkpoints that you\n",
    "# can use with this model, depending on what kind of output you want\n",
    "# See the list of checkpoints: https://github.com/magenta/magenta/tree/master/magenta/models/music_vae#pre-trained-checkpoints\n",
    "!gsutil -q -m cp -R gs://download.magenta.tensorflow.org/models/music_vae/colab2/checkpoints/mel_2bar_big.ckpt.* ./\n",
    "\n",
    "# Import dependencies.\n",
    "from magenta.models.music_vae import configs\n",
    "from magenta.models.music_vae.trained_model import TrainedModel\n",
    "\n",
    "# Initialize the model.\n",
    "print(\"Initializing Music VAE...\")\n",
    "music_vae = TrainedModel(\n",
    "    configs.CONFIG_MAP[\"cat-mel_2bar_big\"],\n",
    "    batch_size=4,\n",
    "    checkpoint_dir_or_path=\"./mel_2bar_big.ckpt\",\n",
    ")\n",
    "\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating new sequences\n",
    "\n",
    "With Music VAE, you can configure how many new sequences to generate, the number of steps the new sequence will be, as well as the \"temperature\" of the result -- the higher the temperature, the more random (and less like the input) your sequence will be. You can play around with these values and see how the resulting sequences are different:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generated_sequences = music_vae.sample(n=2, length=80, temperature=1.0)\n",
    "\n",
    "for ns in generated_sequences:\n",
    "    # print(ns)\n",
    "    note_seq.plot_sequence(ns)\n",
    "    note_seq.play_sequence(ns, synth=note_seq.fluidsynth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpolating between two sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We're going to interpolate between the Twinkle Twinkle Little Star\n",
    "# NoteSequence we defined in the first section, and one of the generated\n",
    "# sequences from the previous VAE example\n",
    "\n",
    "# How many sequences, including the start and end ones, to generate.\n",
    "num_steps = 8\n",
    "\n",
    "# This gives us a list of sequences.\n",
    "note_sequences = music_vae.interpolate(\n",
    "    twinkle_twinkle, teapot, num_steps=num_steps, length=32\n",
    ")\n",
    "\n",
    "# Concatenate them into one long sequence, with the start and\n",
    "# end sequences at each end.\n",
    "interp_seq = note_seq.sequences_lib.concatenate_sequences(note_sequences)\n",
    "\n",
    "note_seq.play_sequence(interp_seq, synth=note_seq.fluidsynth)\n",
    "note_seq.plot_sequence(interp_seq)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8",
   "language": "python",
   "name": "python3.8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}