Experimental_Music_Transformer_Version_2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "ac5a4cf0-d9d2-47b5-9633-b53f8d99a4d2",
          "kernelId": ""
        },
        "id": "SiTIpPjArIyr"
      },
      "source": [
        "# Experimental Music Transformer Version 2 (ver. 0.1)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2023\n",
        "\n",
        "***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "fa0a611c-1803-42ae-bdf6-a49b5a4e781b",
          "kernelId": ""
        },
        "id": "gOd93yV0sGd2"
      },
      "source": [
        "# (SETUP ENVIRONMENT)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RXK4JR0y1uX0"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "gradient": {
          "editing": false,
          "id": "a1a45a91-d909-4fd4-b67a-5e16b971d179",
          "kernelId": ""
        },
        "id": "fX12Yquyuihc"
      },
      "outputs": [],
      "source": [
        "#@title Install all dependencies (run only once per session)\n",
        "\n",
        "!git clone https://github.com/asigalov61/tegridy-tools"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "gradient": {
          "editing": false,
          "id": "b8207b76-9514-4c07-95db-95a4742e52c5",
          "kernelId": ""
        },
        "id": "z7n9vnKmug1J"
      },
      "outputs": [],
      "source": [
        "#@title Import all needed modules\n",
        "\n",
        "print('Loading needed modules. Please wait...')\n",
        "import os\n",
        "\n",
        "import math\n",
        "import statistics\n",
        "import random\n",
        "\n",
        "from collections import Counter\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "if not os.path.exists('/content/Dataset'):\n",
        "    os.makedirs('/content/Dataset')\n",
        "\n",
        "print('Loading TMIDIX module...')\n",
        "os.chdir('/content/tegridy-tools/tegridy-tools')\n",
        "\n",
        "import TMIDIX\n",
        "\n",
        "from joblib import Parallel, delayed, parallel_config\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "os.chdir('/content/')\n",
        "print('Enjoy! :)')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IDybEm0PgkW4"
      },
      "source": [
        "# (DOWNLOAD AND UNZIP DATASETS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UyfROzFdgox_",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "# @title MIDI Dataset\n",
        "%cd /content/Dataset\n",
        "!wget https://github.com/asigalov61/Tegridy-MIDI-Dataset/raw/master/Mono-Melodies/Piano-Violin/Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip.001\n",
        "!wget https://github.com/asigalov61/Tegridy-MIDI-Dataset/raw/master/Mono-Melodies/Piano-Violin/Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip.002\n",
        "!cat Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip* > Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip\n",
        "!unzip Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip\n",
        "!rm Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip.001\n",
        "!rm Mono-Melodies-Piano-Violin-CC-BY-NC-SA.zip.002\n",
        "%cd /content/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JwrqQeie08t0"
      },
      "source": [
        "# (FILE LIST)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FLxHvO-wlwfU"
      },
      "source": [
        "# (PROCESS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HkS8pYJBPSFN",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title TMIDIX MIDI Processor\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading TMIDIX MIDI Processor...')\n",
        "print('=' * 70)\n",
        "\n",
        "def TMIDIX_MIDI_Processor(midi_file):\n",
        "\n",
        "    melody_chords = []\n",
        "\n",
        "    try:\n",
        "\n",
        "        fn = os.path.basename(midi_file)\n",
        "\n",
        "        # Filtering out GIANT4 MIDIs\n",
        "        file_size = os.path.getsize(midi_file)\n",
        "\n",
        "        if file_size <= 1000000:\n",
        "\n",
        "          #=======================================================\n",
        "          # START PROCESSING\n",
        "\n",
        "          # Convering MIDI to ms score with MIDI.py module\n",
        "          score = TMIDIX.midi2single_track_ms_score(open(midi_file, 'rb').read(), recalculate_channels=False)\n",
        "\n",
        "          # INSTRUMENTS CONVERSION CYCLE\n",
        "          events_matrix = []\n",
        "          itrack = 1\n",
        "          patches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "\n",
        "          while itrack < len(score):\n",
        "              for event in score[itrack]:\n",
        "                  if event[0] == 'note' or event[0] == 'patch_change':\n",
        "                      events_matrix.append(event)\n",
        "              itrack += 1\n",
        "\n",
        "          events_matrix.sort(key=lambda x: x[1])\n",
        "\n",
        "          events_matrix1 = []\n",
        "\n",
        "          for event in events_matrix:\n",
        "                  if event[0] == 'patch_change':\n",
        "                        patches[event[2]] = event[3]\n",
        "\n",
        "                  if event[0] == 'note':\n",
        "                        event.extend([patches[event[3]]])\n",
        "\n",
        "                        if events_matrix1:\n",
        "                            if (event[1] == events_matrix1[-1][1]):\n",
        "                                if ([event[3], event[4]] != events_matrix1[-1][3:5]):\n",
        "                                    events_matrix1.append(event)\n",
        "                            else:\n",
        "                                events_matrix1.append(event)\n",
        "\n",
        "                        else:\n",
        "                            events_matrix1.append(event)\n",
        "\n",
        "        if len(events_matrix1) > 0:\n",
        "            if min([e[1] for e in events_matrix1]) >= 0 and min([e[2] for e in events_matrix1]) >= 0:\n",
        "\n",
        "                #=======================================================\n",
        "                # PRE-PROCESSING\n",
        "\n",
        "                # checking number of instruments in a composition\n",
        "                instruments_list = list(set([y[3] for y in events_matrix1]))\n",
        "\n",
        "                if len(events_matrix1) > 0:\n",
        "\n",
        "                    #===================================\n",
        "                    # ORIGINAL COMPOSITION\n",
        "                    #===================================\n",
        "\n",
        "                    # Adjusting timings\n",
        "\n",
        "                    for e in events_matrix1:\n",
        "                      e[1] = int(e[1] / 16)\n",
        "                      e[2] = int(e[2] / 16)\n",
        "\n",
        "                    # Sorting by patch, pitch, then by start-time\n",
        "\n",
        "                    events_matrix1.sort(key=lambda x: x[6])\n",
        "                    events_matrix1.sort(key=lambda x: x[4], reverse=True)\n",
        "                    events_matrix1.sort(key=lambda x: x[1])\n",
        "\n",
        "                    #=======================================================\n",
        "                    # FINAL PROCESSING\n",
        "\n",
        "                    #=======================================================\n",
        "                    # MAIN PROCESSING CYCLE\n",
        "                    #=======================================================\n",
        "\n",
        "                    pe = events_matrix1[0]\n",
        "\n",
        "                    notes = []\n",
        "\n",
        "                    for e in events_matrix1:\n",
        "\n",
        "                      time = max(0, min(127, (e[1] - pe[1])))\n",
        "                      dur = max(0, min(127, e[2]))\n",
        "                      cha = max(0, min(15, e[3]))\n",
        "                      ptc = max(1, min(127, e[4]))\n",
        "\n",
        "                      if cha == 3:\n",
        "                        cha = 1\n",
        "\n",
        "                      notes.append([time, dur, cha, ptc])\n",
        "\n",
        "                      pe = e\n",
        "\n",
        "                    return notes\n",
        "\n",
        "    except:\n",
        "      return None\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "1s_9bVGrPU_q"
      },
      "outputs": [],
      "source": [
        "#@title Save file list\n",
        "###########\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading MIDI files...')\n",
        "print('This may take a while on a large dataset in particular.')\n",
        "\n",
        "dataset_addr = \"/content/Dataset\"\n",
        "\n",
        "# os.chdir(dataset_addr)\n",
        "filez = list()\n",
        "for (dirpath, dirnames, filenames) in os.walk(dataset_addr):\n",
        "    filez += [os.path.join(dirpath, file) for file in filenames]\n",
        "print('=' * 70)\n",
        "\n",
        "if not filez:\n",
        "    print('Could not find any MIDI files. Please check Dataset dir...')\n",
        "    print('=' * 70)\n",
        "\n",
        "else:\n",
        "  print('Randomizing file list...')\n",
        "  random.shuffle(filez)\n",
        "  print('Done!')\n",
        "  print('=' * 70)\n",
        "  print('Total files:', len(filez))\n",
        "  print('=' * 70)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nFkq_d4nPVmW",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Process MIDIs with TMIDIX MIDI processor\n",
        "\n",
        "print('=' * 70)\n",
        "print('TMIDIX MIDI Processor')\n",
        "print('=' * 70)\n",
        "print('Starting up...')\n",
        "print('=' * 70)\n",
        "\n",
        "###########\n",
        "\n",
        "melody_chords_f = []\n",
        "\n",
        "print('Processing MIDI files. Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "for i in tqdm(range(0, len(filez), 16)):\n",
        "\n",
        "  with parallel_config(backend='threading', n_jobs=16, verbose = 0):\n",
        "\n",
        "    output = Parallel()(delayed(TMIDIX_MIDI_Processor)(f) for f in filez[i:i+16])\n",
        "\n",
        "    for o in output:\n",
        "\n",
        "        if o is not None:\n",
        "            melody_chords_f.append(o)\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "foSl_4jL-Hfc"
      },
      "outputs": [],
      "source": [
        "melody_chords_f[1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zjGwEKInQT8a"
      },
      "outputs": [],
      "source": [
        "TMIDIX.Tegridy_Any_Pickle_File_Writer(melody_chords_f, '/content/Processed_MIDIs')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lZGSjKe9QUey"
      },
      "outputs": [],
      "source": [
        "melody_chords_f = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/Processed_MIDIs')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pvcxH5_CHwLZ",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Test INTs\n",
        "\n",
        "train_data1 = melody_chords_f[4]\n",
        "\n",
        "#train_data1 = max(melody_chords_f, key = len)\n",
        "\n",
        "print('Sample INTs', train_data1[:15])\n",
        "\n",
        "out = train_data1\n",
        "\n",
        "patches = [0] * 16\n",
        "patches[1] = 40\n",
        "\n",
        "if len(out) != 0:\n",
        "\n",
        "    song = out\n",
        "    song_f = []\n",
        "\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 90\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "\n",
        "    for s in song:\n",
        "\n",
        "\n",
        "        time += s[0] * 16\n",
        "        dur = s[1] * 16\n",
        "        channel = s[2]\n",
        "        pitch = s[3]\n",
        "\n",
        "\n",
        "        song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "\n",
        "\n",
        "detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                          output_signature = 'Experimental Music Transformer',\n",
        "                                                          output_file_name = '/content/Experimental-Music-Transformer-Composition',\n",
        "                                                          track_name='Project Los Angeles',\n",
        "                                                          list_of_MIDI_patches=patches\n",
        "                                                          )\n",
        "\n",
        "print('Done!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5I9hQnf51uX4"
      },
      "outputs": [],
      "source": [
        "len(melody_chords_f[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4hlZohOTSKqq"
      },
      "source": [
        "# (TRAIN DATA)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EMX0ynkxPtGH"
      },
      "outputs": [],
      "source": [
        "train_data = []\n",
        "\n",
        "for m in tqdm.tqdm(melody_chords_f):\n",
        "\n",
        "    cha = m[0][2]\n",
        "\n",
        "    dat = [1025, ((cha * 128) + m[0][3])+640, 0]\n",
        "\n",
        "    for mm in m:\n",
        "\n",
        "        cha = mm[2]\n",
        "\n",
        "        if mm[0] != 0:\n",
        "            dat.extend([mm[0], mm[1]+128, ((cha * 128) + mm[3])+256])\n",
        "        else:\n",
        "            dat.extend([mm[1]+128, ((cha * 128) + mm[3])+256])\n",
        "\n",
        "    dat = dat[:1025]\n",
        "\n",
        "    ids = [] # 0 - 256 and 640 - 1024\n",
        "    nums = [] # 256 - 640\n",
        "    masks = [] # 1024\n",
        "\n",
        "    for d in dat:\n",
        "      if 0 <= d < 256:\n",
        "        ids.append(d)\n",
        "        nums.append(-1)\n",
        "        masks.append(False)\n",
        "\n",
        "      if 256 <= d < 640:\n",
        "        ids.append(1024)\n",
        "        nums.append(d)\n",
        "        masks.append(True)\n",
        "\n",
        "    ids += [1026] * (1025 - len(ids))\n",
        "    nums += [-1] * (1025 - len(nums))\n",
        "    masks += [False] * (1025 - len(masks))\n",
        "\n",
        "    train_data.append([ids, nums, masks])\n",
        "\n",
        "# Total dict size 1027\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CNhswCvd1uX5"
      },
      "outputs": [],
      "source": [
        "len(train_data), max(train_data, key=len) == min(train_data, key=len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2zIalzV9Ego9"
      },
      "outputs": [],
      "source": [
        "train_data[555][:8]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "19WxN9LTZjLX"
      },
      "outputs": [],
      "source": [
        "random.shuffle(train_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z-omEbFmVlOI"
      },
      "outputs": [],
      "source": [
        "TMIDIX.Tegridy_Any_Pickle_File_Writer(train_data, '/content/INTs')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R7FtGA4lZ8ub"
      },
      "outputs": [],
      "source": [
        "train_data = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/INTs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RtbJjoztholN"
      },
      "outputs": [],
      "source": [
        "len(max(train_data[0], key=len)), len(min(train_data[0], key=len))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (TRAIN MODEL)"
      ],
      "metadata": {
        "id": "hQ5AZV619XnV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tLOe9HaMvXLq"
      },
      "outputs": [],
      "source": [
        "!pip install x-transformers\n",
        "!pip install einops\n",
        "!pip install torch-summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x19qtRenwMq2"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "torch.__version__"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M_jXcWykwLuS"
      },
      "outputs": [],
      "source": [
        "# Load modules and make data dir\n",
        "\n",
        "print('Loading modules...')\n",
        "\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import secrets\n",
        "import tqdm\n",
        "import math\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from torchsummary import summary\n",
        "from sklearn import metrics\n",
        "\n",
        "%cd /content/tegridy-tools/tegridy-tools/\n",
        "\n",
        "import TMIDIX\n",
        "\n",
        "%cd /content/tegridy-tools/tegridy-tools/X-Transformer\n",
        "\n",
        "from x_transformers import (\n",
        "    Decoder,\n",
        "    XValTransformerWrapper,\n",
        "    XValAutoregressiveWrapper\n",
        ")\n",
        "\n",
        "torch.set_float32_matmul_precision('high')\n",
        "torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul\n",
        "torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn\n",
        "\n",
        "%cd /content/\n",
        "\n",
        "if not os.path.exists('/content/INTS'):\n",
        "    os.makedirs('/content/INTS')\n",
        "\n",
        "import random\n",
        "\n",
        "print('Done')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/INTs')"
      ],
      "metadata": {
        "id": "KyYArgS27kyN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AXN_O2dCxqny"
      },
      "outputs": [],
      "source": [
        "len(train_data) // 8"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YVYD6nEGwU8z"
      },
      "outputs": [],
      "source": [
        "# @title Setup and init the model\n",
        "\n",
        "# constants\n",
        "\n",
        "SEQ_LEN = 8192 # Models seq len\n",
        "PAD_IDX = 1026 # Models pad index\n",
        "\n",
        "BATCH_SIZE = 4\n",
        "NUM_EPOCHS = 100\n",
        "GRADIENT_ACCUMULATE_EVERY = 4\n",
        "\n",
        "\n",
        "LEARNING_RATE = 2e-4\n",
        "\n",
        "VALIDATE_EVERY  = 100\n",
        "SAVE_EVERY = 500\n",
        "GENERATE_EVERY  = 100\n",
        "PRINT_STATS_EVERY = 20\n",
        "\n",
        "GENERATE_LENGTH = 32\n",
        "\n",
        "# helpers\n",
        "\n",
        "def cycle(loader):\n",
        "    while True:\n",
        "        for data in loader:\n",
        "            yield data\n",
        "\n",
        "# instantiate the model\n",
        "\n",
        "model = XValTransformerWrapper(\n",
        "    num_tokens = 1027,\n",
        "    numerical_token_id = 1024,\n",
        "    max_seq_len = 1024,\n",
        "    attn_layers = Decoder(\n",
        "        dim = 1024,\n",
        "        depth = 8,\n",
        "        heads = 8,\n",
        "\n",
        "    )\n",
        ")\n",
        "\n",
        "# wrap it with the xval autoregressive wrapper\n",
        "\n",
        "model = XValAutoregressiveWrapper(model, ignore_index=PAD_IDX)\n",
        "\n",
        "model.cuda()\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "summary(model)\n",
        "\n",
        "# Dataloader\n",
        "\n",
        "class MusicDataset(Dataset):\n",
        "    def __init__(self, data, seq_len):\n",
        "        super().__init__()\n",
        "        self.data = data\n",
        "        self.seq_len = seq_len\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "\n",
        "        ids = torch.Tensor(self.data[index][0][:self.seq_len+1]).long()\n",
        "        nums = torch.Tensor(self.data[index][1][:self.seq_len+1]).long()\n",
        "        masks = torch.Tensor(self.data[index][2][:self.seq_len+1]).bool()\n",
        "\n",
        "        return ids.cuda(), nums.cuda(), masks.cuda()\n",
        "\n",
        "    def __len__(self):\n",
        "        return (len(self.data) // BATCH_SIZE) * BATCH_SIZE\n",
        "\n",
        "# precision/optimizer/scaler\n",
        "\n",
        "dtype = torch.float16\n",
        "\n",
        "ctx = torch.amp.autocast(device_type='cuda', dtype=dtype)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "scaler = torch.cuda.amp.GradScaler(enabled=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random.shuffle(train_data)\n",
        "\n",
        "train_dataset = MusicDataset(train_data, SEQ_LEN)\n",
        "val_dataset   = MusicDataset(train_data, SEQ_LEN)\n",
        "train_loader  = DataLoader(train_dataset, batch_size = BATCH_SIZE)\n",
        "val_loader    = DataLoader(val_dataset, batch_size = BATCH_SIZE)"
      ],
      "metadata": {
        "id": "_UYFdm2BiLIJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p0viaFQJwzaf"
      },
      "outputs": [],
      "source": [
        "# @title Train the model\n",
        "torch.cuda.empty_cache()\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "\n",
        "train_accs = []\n",
        "val_accs = []\n",
        "\n",
        "nsteps = 0\n",
        "\n",
        "PRINT_STATS_EVERY = 200\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):  # replace NUM_EPOCHS with the actual number of epochs\n",
        "\n",
        "      print('=' * 70)\n",
        "      print('Epoch #', epoch)\n",
        "      print('=' * 70)\n",
        "      model.train()  # set the model to training mode\n",
        "      total_loss = 0\n",
        "      optimizer.zero_grad(set_to_none=True)  # Initialize gradients to zero at the start of the epoch\n",
        "\n",
        "      for batch_idx, batch in enumerate(tqdm.tqdm(train_loader)):  # iterate over batches of data\n",
        "          ids, nums, masks = batch  # unpack the source and target tensors from the current batch\n",
        "\n",
        "          with torch.cuda.amp.autocast():\n",
        "              loss = model(ids, nums, mask=masks)  # forward pass\n",
        "\n",
        "          # loss = loss / GRADIENT_ACCUMULATE_EVERY  # Normalize the loss by the number of accumulation steps\n",
        "          # scaler.scale(loss).backward()  # Backward pass with gradient scaling\n",
        "\n",
        "          train_losses.append(loss.mean().item() * GRADIENT_ACCUMULATE_EVERY)\n",
        "          # train_accs.append(acc.mean().item())\n",
        "\n",
        "\n",
        "          if (batch_idx + 1) % GRADIENT_ACCUMULATE_EVERY == 0:  # Perform optimization step after accumulating gradients\n",
        "              # scaler.unscale_(optimizer)\n",
        "              torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "              #scaler.step(optimizer)\n",
        "              #scaler.update()\n",
        "              optimizer.step()\n",
        "              optimizer.zero_grad(set_to_none=True)  # Reset gradients after optimization step\n",
        "\n",
        "          total_loss += loss.item() * GRADIENT_ACCUMULATE_EVERY  # Undo the normalization for logging\n",
        "\n",
        "          if nsteps % PRINT_STATS_EVERY == 0:\n",
        "              # print(f'Training Loss: {total_loss / (batch_idx + 1)}, Accuracy: {acc.item()}')\n",
        "              print(f'Training Loss: {total_loss / (batch_idx + 1)}')\n",
        "\n",
        "\n",
        "\n",
        "      nsteps += 1\n",
        "\n",
        "      '''if i % VALIDATE_EVERY == 0:\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "          with ctx:\n",
        "            val_loss, val_acc = model(next(val_loader))\n",
        "\n",
        "            print(f'Validation loss: {val_loss.mean().item()}')\n",
        "            print(f'Validation acc: {val_acc.mean().item()}')\n",
        "\n",
        "            val_losses.append(val_loss.mean().item())\n",
        "            val_accs.append(val_acc.mean().item())\n",
        "\n",
        "            print('Plotting training loss graph...')\n",
        "\n",
        "            tr_loss_list = train_losses\n",
        "            plt.plot([i for i in range(len(tr_loss_list))] ,tr_loss_list, 'b')\n",
        "            plt.show()\n",
        "            plt.close()\n",
        "            print('Done!')\n",
        "\n",
        "            print('Plotting training acc graph...')\n",
        "\n",
        "            tr_loss_list = train_accs\n",
        "            plt.plot([i for i in range(len(tr_loss_list))] ,tr_loss_list, 'b')\n",
        "            plt.show()\n",
        "            plt.close()\n",
        "            print('Done!')\n",
        "\n",
        "            print('Plotting validation loss graph...')\n",
        "            tr_loss_list = val_losses\n",
        "            plt.plot([i for i in range(len(tr_loss_list))] ,tr_loss_list, 'b')\n",
        "            plt.show()\n",
        "            plt.close()\n",
        "            print('Done!')\n",
        "\n",
        "            print('Plotting validation acc graph...')\n",
        "            tr_loss_list = val_accs\n",
        "            plt.plot([i for i in range(len(tr_loss_list))] ,tr_loss_list, 'b')\n",
        "            plt.show()\n",
        "            plt.close()\n",
        "            print('Done!')'''\n",
        "\n",
        "      '''if i % GENERATE_EVERY == 0:\n",
        "        model.eval()\n",
        "\n",
        "        inp = random.choice(val_dataset)[:-1]\n",
        "\n",
        "        print(inp)\n",
        "\n",
        "        with ctx:\n",
        "\n",
        "            sample = model.generate(inp[None, ...], GENERATE_LENGTH)\n",
        "\n",
        "        print(sample)'''\n",
        "\n",
        "      if i % SAVE_EVERY == 0:\n",
        "\n",
        "          print('Saving model progress. Please wait...')\n",
        "          print('model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth')\n",
        "\n",
        "          fname = '/content/model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth'\n",
        "\n",
        "          torch.save(model.state_dict(), fname)\n",
        "\n",
        "          data = [train_losses, train_accs, val_losses, val_accs]\n",
        "\n",
        "          TMIDIX.Tegridy_Any_Pickle_File_Writer(data, '/content/losses_accs')\n",
        "\n",
        "          print('Done!')\n",
        "\n",
        "#======================================================================================================\n",
        "\n",
        "print('Saving model progress. Please wait...')\n",
        "print('model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth')\n",
        "\n",
        "fname = '/content/model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth'\n",
        "\n",
        "torch.save(model.state_dict(), fname)\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "data = [train_losses, train_accs, val_losses, val_accs]\n",
        "\n",
        "TMIDIX.Tegridy_Any_Pickle_File_Writer(data, '/content/losses_accuracies')\n",
        "\n",
        "# Save training loss graph\n",
        "\n",
        "plt.plot([i for i in range(len(train_losses))] ,train_losses, 'b')\n",
        "plt.savefig('/content/training_loss_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save training acc graph\n",
        "\n",
        "plt.plot([i for i in range(len(train_accs))] ,train_accs, 'b')\n",
        "plt.savefig('/content/training_acc_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save validation loss graph\n",
        "\n",
        "plt.plot([i for i in range(len(val_losses))] ,val_losses, 'b')\n",
        "plt.savefig('/content/validation_loss_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save validation acc graph\n",
        "\n",
        "plt.plot([i for i in range(len(val_accs))] ,val_accs, 'b')\n",
        "plt.savefig('/content/validation_acc_graph.png')\n",
        "plt.close()\n",
        "print('Done!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRw_eBMDbYhl"
      },
      "outputs": [],
      "source": [
        "print('Saving model progress. Please wait...')\n",
        "print('model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth')\n",
        "\n",
        "fname = '/content/model_checkpoint_' + str(nsteps) + '_steps_' + str(round(float(train_losses[-1]), 4)) + '_loss_' + str(round(float(train_accs[-1]), 4)) + '_acc.pth'\n",
        "\n",
        "torch.save(model.state_dict(), fname)\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "data = [train_losses, train_accs, val_losses, val_accs]\n",
        "\n",
        "TMIDIX.Tegridy_Any_Pickle_File_Writer(data, '/content/losses_accuracies')\n",
        "\n",
        "# Save training loss graph\n",
        "\n",
        "plt.plot([i for i in range(len(train_losses))] ,train_losses, 'b')\n",
        "plt.savefig('/content/training_loss_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save training acc graph\n",
        "\n",
        "plt.plot([i for i in range(len(train_accs))] ,train_accs, 'b')\n",
        "plt.savefig('/content/training_acc_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save validation loss graph\n",
        "\n",
        "plt.plot([i for i in range(len(val_losses))] ,val_losses, 'b')\n",
        "plt.savefig('/content/validation_loss_graph.png')\n",
        "plt.close()\n",
        "print('Done!')\n",
        "\n",
        "# Save validation acc graph\n",
        "\n",
        "plt.plot([i for i in range(len(val_accs))] ,val_accs, 'b')\n",
        "plt.savefig('/content/validation_acc_graph.png')\n",
        "plt.close()\n",
        "print('Done!')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EjMO6u5N1uX7"
      },
      "source": [
        "# EVAL"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g1dL8gUT1uX8"
      },
      "outputs": [],
      "source": [
        "dtype = 'float16'\n",
        "device_type = 'cuda'\n",
        "ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]\n",
        "ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eIIqeaipzVil"
      },
      "outputs": [],
      "source": [
        "model.eval()\n",
        "\n",
        "x = torch.tensor(train_data[2][:900], dtype=torch.long, device='cuda')[None, ...]\n",
        "#x = torch.tensor([[1024]] * 1, dtype=torch.long, device='cuda')\n",
        "\n",
        "# run generation\n",
        "\n",
        "with ctx:\n",
        "    out = model.generate(x,\n",
        "                        1023,\n",
        "                        temperature=1,\n",
        "                        return_prime=False,\n",
        "                        verbose=True)\n",
        "\n",
        "y = out.tolist()\n",
        "\n",
        "print('---------------')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6uBfCT2JbYeD"
      },
      "outputs": [],
      "source": [
        "print(y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E29jsu4SzZNw",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Test INTs\n",
        "\n",
        "train_data1 = out3 # y[0]\n",
        "\n",
        "#train_data1 = max(melody_chords_f, key = len)\n",
        "\n",
        "print('Sample INTs', train_data1[:15])\n",
        "\n",
        "out = train_data1\n",
        "\n",
        "patches = [0] * 16\n",
        "patches[3] = 40\n",
        "\n",
        "if len(out) != 0:\n",
        "\n",
        "    song = out\n",
        "    song_f = []\n",
        "\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 90\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    for ss in tqdm.tqdm(song):\n",
        "\n",
        "        if 0 <= ss < 256:\n",
        "\n",
        "            time += (ss * 16)\n",
        "\n",
        "        if 256 <= ss < 512:\n",
        "\n",
        "            dur = (ss-256) * 16\n",
        "\n",
        "        if 512 <= ss < 640:\n",
        "\n",
        "            pitch = ss-512\n",
        "\n",
        "        if 640 <= ss < 642:\n",
        "\n",
        "            channel = ss-640\n",
        "\n",
        "            if channel == 1:\n",
        "                channel = 3\n",
        "\n",
        "        if 642 <= ss < 770:\n",
        "            vel = ss-642\n",
        "\n",
        "            song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                          output_signature = 'Experimental Music Transformer',\n",
        "                                                          output_file_name = '/content/Experimental-Music-Transformer-Composition',\n",
        "                                                          track_name='Project Los Angeles',\n",
        "                                                          list_of_MIDI_patches=patches\n",
        "                                                          )\n",
        "\n",
        "print('Done!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8745GbH2zYpZ"
      },
      "outputs": [],
      "source": [
        "tok_emb = model.net.token_emb.emb.weight.detach().cpu().tolist()\n",
        "\n",
        "cos_sim = metrics.pairwise_distances(\n",
        "  tok_emb, metric='cosine'\n",
        ")\n",
        "plt.figure(figsize=(7, 7))\n",
        "plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"nearest\")\n",
        "im_ratio = cos_sim.shape[0] / cos_sim.shape[1]\n",
        "plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "plt.xlabel(\"Position\")\n",
        "plt.ylabel(\"Position\")\n",
        "plt.tight_layout()\n",
        "plt.plot()\n",
        "plt.savefig(\"/content/Experimental-Music-Transformer-Tokens-Embeddings-Plot.png\", bbox_inches=\"tight\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Congrats! You did it! :)"
      ],
      "metadata": {
        "id": "lI0FIbks9bqs"
      }
    }
  ],
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}