models_src/model_c_transformer/MuGenTransformer-v3-stable-export.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## V3\n",
    "V1\n",
    "- custom generator\n",
    "- to load random sequence from each batch\n",
    "- Result is very positive :D\n",
    "-----\n",
    "V2\n",
    "- Testing out for different preprocess data type: 100f data\n",
    "- After 6000 epoches, the results seem to be quite positive, although the data takes quite long to generate\n",
    "- Unfortunately, the model overfitted very badly. \n",
    "- Also tested with different encoding method. However, model performed really badly.thus thus was the end of v2\n",
    "-----\n",
    "V3\n",
    "- revert back to v1 with modification.\n",
    "- will include tags: < UNK >, < PAD >"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_kkk3XW3p_Vl",
    "outputId": "cc236305-ad5e-478a-fa09-1fe9b030a070"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ./.cache/pip/wheels/1e/a3/bc/889b2ea8a6d6561f015632961a3a7d492e4bfe2c21edc37c67/music21-6.3.0-py3-none-any.whl\n",
      "Requirement already satisfied: chardet in /opt/conda/lib/python3.6/site-packages (from music21) (3.0.4)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from music21) (0.15.1)\n",
      "Collecting webcolors\n",
      "  Using cached webcolors-1.11.1-py3-none-any.whl (9.9 kB)\n",
      "Requirement already satisfied: more-itertools in /opt/conda/lib/python3.6/site-packages (from music21) (8.3.0)\n",
      "Installing collected packages: webcolors, music21\n",
      "Successfully installed music21-6.3.0 webcolors-1.11.1\n",
      "Collecting mido\n",
      "  Using cached mido-1.2.9-py2.py3-none-any.whl (52 kB)\n",
      "Installing collected packages: mido\n",
      "Successfully installed mido-1.2.9\n",
      "Processing ./.cache/pip/wheels/b9/ef/f5/9f35c0da899320e8f443f44ecbfa3a6721a4c3eacccad39844/pretty_midi-0.2.9-py3-none-any.whl\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from pretty_midi) (1.15.0)\n",
      "Requirement already satisfied: numpy>=1.7.0 in /opt/conda/lib/python3.6/site-packages (from pretty_midi) (1.18.5)\n",
      "Requirement already satisfied: mido>=1.1.16 in /opt/conda/lib/python3.6/site-packages (from pretty_midi) (1.2.9)\n",
      "Installing collected packages: pretty-midi\n",
      "Successfully installed pretty-midi-0.2.9\n"
     ]
    }
   ],
   "source": [
    "!pip install music21\n",
    "!pip install mido\n",
    "!pip install pretty_midi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7iw9iWCXjLhJ"
   },
   "source": [
    "### Import module and define path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "N0dfe-nxqbWl"
   },
   "outputs": [],
   "source": [
    "import pretty_midi\n",
    "from music21 import *\n",
    "import numpy as np\n",
    "import matplotlib.patches as patches\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import glob\n",
    "from itertools import groupby\n",
    "import math\n",
    "import random\n",
    "import pickle\n",
    "# from keras.preprocessing import sequence\n",
    "# from keras.models import Sequential \n",
    "# from keras.layers import Dense, LSTM, Bidirectional, Dropout, GlobalMaxPooling1D, Activation, GlobalMaxPooling2D\n",
    "# from keras_self_attention import SeqSelfAttention\n",
    "# from keras.utils import to_categorical\n",
    "# from keras.callbacks import ModelCheckpoint\n",
    "# from keras.layers import Layer\n",
    "# from tensorflow.python.client import device_lib\n",
    "# import keras\n",
    "import gc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "import random\n",
    "\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "mlb = MultiLabelBinarizer()\n",
    "mlb.fit([np.arange(128).tolist()])\n",
    "\n",
    "# data_path = \"./selected_data/\"\n",
    "# data_path = \"./doug_mckenzie_midi/\"\n",
    "# data_path = \"./sample_data/\"\n",
    "# encoded_data_path = \"./data/encoded_doug_mckenzie_midi_100f/\"\n",
    "encoded_data_path = \"./encoded_doug_mckenzie_midi_16_v2/\"\n",
    "# encoded_data_path = \"../data/encoded_doug_mckenzie_midi_32_v2/\"\n",
    "# encoded_data_path = \"./data/encoded_doug_mckenzie_midi_32/\"\n",
    "output_path = \"./output/\"\n",
    "\n",
    "batch_size = 32\n",
    "# sequence_length = 500\n",
    "sequence_length = 600\n",
    "generate_sample_every_ep = 100\n",
    "\n",
    "maxlen = sequence_length  # Max sequence size\n",
    "# embed_dim = 128  # Embedding size for each token\n",
    "embed_dim = 128  # Embedding size for each token\n",
    "num_heads = 4  # Number of attention heads\n",
    "feed_forward_dim = 128  # Hidden layer size in feed forward network inside transformer\n",
    "\n",
    "combi_to_int_pickle = \"combi_to_int.pickle\"\n",
    "int_to_combi_pickle = \"int_to_combi.pickle\"\n",
    "vocab_pickle = \"vocab.pickle\"\n",
    "\n",
    "# vocab_size = 50000\n",
    "vocab_size = 40000\n",
    "unk_tag_str = '<UNK>'\n",
    "unk_tag_idx = 0\n",
    "pad_tag_str = ''\n",
    "pad_tag_idx = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nY848lUWznQ2"
   },
   "source": [
    "## Preprocess Data\n",
    "In order to carry out this step you need to unzip the encoded_doug_mckenzie_midi_16_v2.zip. Or you can just import all the varibales from the pickle files in the import step\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "og1gG5GvzpbK"
   },
   "outputs": [],
   "source": [
    "all_songs = []\n",
    "# all_song_in_tuple = []\n",
    "all_songs_np = np.empty((0,128), np.int8)\n",
    "for temp in glob.glob(encoded_data_path + \"*.npy\"):\n",
    "    encoded_data = np.load(temp).astype(np.int8)\n",
    "    all_songs.append(encoded_data)\n",
    "    all_songs_np = np.append(all_songs_np, encoded_data, axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for song in all_song_tokenised:\n",
    "    print(len(song))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(356057, 128)\n",
      "vocab size: 40000\n",
      "vocab first 5 words: ['<UNK>', '', (), (72,), (67,)]\n"
     ]
    }
   ],
   "source": [
    "print(all_songs_np.shape)\n",
    "unique_np, counts = np.unique(all_songs_np, axis=0, return_counts=True)\n",
    "\n",
    "unique_note_intergerized = np.array(mlb.inverse_transform(unique_np))\n",
    "count_sort_ind = np.argsort(-counts)\n",
    "\n",
    "vocab = unique_note_intergerized[count_sort_ind][:vocab_size-2].tolist()\n",
    "top_counts = counts[count_sort_ind][:vocab_size-1].tolist()\n",
    "# vocab.insert(unk_tag_idx, unk_tag_str)\n",
    "\n",
    "vocab.sort(key=len)\n",
    "# vocab = unique_note_intergerized\n",
    "vocab.insert(unk_tag_idx, unk_tag_str)\n",
    "vocab.insert(pad_tag_idx, pad_tag_str)\n",
    "vocab_size = len(vocab)\n",
    "# vocab_size = 54000\n",
    "print(f\"vocab size: {len(vocab)}\")\n",
    "print(f\"vocab first 5 words: {vocab[:5]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing song number 0\n",
      "processing song number 1\n",
      "processing song number 2\n",
      "processing song number 3\n",
      "processing song number 4\n",
      "processing song number 5\n",
      "processing song number 6\n",
      "processing song number 7\n",
      "processing song number 8\n",
      "processing song number 9\n",
      "processing song number 10\n",
      "processing song number 11\n",
      "processing song number 12\n",
      "processing song number 13\n",
      "processing song number 14\n",
      "processing song number 15\n",
      "processing song number 16\n",
      "processing song number 17\n",
      "processing song number 18\n",
      "processing song number 19\n",
      "processing song number 20\n",
      "processing song number 21\n",
      "processing song number 22\n",
      "processing song number 23\n",
      "processing song number 24\n",
      "processing song number 25\n",
      "processing song number 26\n",
      "processing song number 27\n",
      "processing song number 28\n",
      "processing song number 29\n",
      "processing song number 30\n",
      "processing song number 31\n",
      "processing song number 32\n",
      "processing song number 33\n",
      "processing song number 34\n",
      "processing song number 35\n",
      "processing song number 36\n",
      "processing song number 37\n",
      "processing song number 38\n",
      "processing song number 39\n",
      "processing song number 40\n",
      "processing song number 41\n",
      "processing song number 42\n",
      "processing song number 43\n",
      "processing song number 44\n",
      "processing song number 45\n",
      "processing song number 46\n",
      "processing song number 47\n",
      "processing song number 48\n",
      "processing song number 49\n",
      "processing song number 50\n",
      "processing song number 51\n",
      "processing song number 52\n",
      "processing song number 53\n",
      "processing song number 54\n",
      "processing song number 55\n",
      "processing song number 56\n",
      "processing song number 57\n",
      "processing song number 58\n",
      "processing song number 59\n",
      "processing song number 60\n",
      "processing song number 61\n",
      "processing song number 62\n",
      "processing song number 63\n",
      "processing song number 64\n",
      "processing song number 65\n",
      "processing song number 66\n",
      "processing song number 67\n",
      "processing song number 68\n",
      "processing song number 69\n",
      "processing song number 70\n",
      "processing song number 71\n",
      "processing song number 72\n",
      "processing song number 73\n",
      "processing song number 74\n",
      "processing song number 75\n",
      "processing song number 76\n",
      "processing song number 77\n",
      "processing song number 78\n",
      "processing song number 79\n",
      "processing song number 80\n",
      "processing song number 81\n",
      "processing song number 82\n",
      "processing song number 83\n",
      "processing song number 84\n",
      "processing song number 85\n",
      "processing song number 86\n",
      "processing song number 87\n",
      "processing song number 88\n",
      "processing song number 89\n",
      "processing song number 90\n",
      "processing song number 91\n",
      "processing song number 92\n",
      "processing song number 93\n",
      "processing song number 94\n",
      "processing song number 95\n",
      "processing song number 96\n",
      "processing song number 97\n",
      "processing song number 98\n",
      "processing song number 99\n",
      "processing song number 100\n",
      "processing song number 101\n",
      "processing song number 102\n",
      "processing song number 103\n",
      "processing song number 104\n",
      "processing song number 105\n",
      "processing song number 106\n",
      "processing song number 107\n",
      "processing song number 108\n",
      "processing song number 109\n",
      "processing song number 110\n",
      "processing song number 111\n",
      "processing song number 112\n",
      "processing song number 113\n",
      "processing song number 114\n",
      "processing song number 115\n",
      "processing song number 116\n",
      "processing song number 117\n",
      "processing song number 118\n",
      "processing song number 119\n",
      "processing song number 120\n",
      "processing song number 121\n",
      "processing song number 122\n",
      "processing song number 123\n",
      "processing song number 124\n",
      "processing song number 125\n",
      "processing song number 126\n",
      "processing song number 127\n",
      "processing song number 128\n",
      "processing song number 129\n",
      "processing song number 130\n",
      "processing song number 131\n",
      "processing song number 132\n",
      "processing song number 133\n",
      "processing song number 134\n",
      "processing song number 135\n",
      "processing song number 136\n",
      "processing song number 137\n",
      "processing song number 138\n",
      "processing song number 139\n",
      "processing song number 140\n",
      "processing song number 141\n",
      "processing song number 142\n",
      "processing song number 143\n",
      "processing song number 144\n",
      "processing song number 145\n",
      "processing song number 146\n",
      "processing song number 147\n",
      "processing song number 148\n",
      "processing song number 149\n",
      "processing song number 150\n",
      "processing song number 151\n",
      "processing song number 152\n",
      "processing song number 153\n",
      "processing song number 154\n",
      "processing song number 155\n",
      "processing song number 156\n",
      "processing song number 157\n",
      "processing song number 158\n",
      "processing song number 159\n",
      "processing song number 160\n",
      "processing song number 161\n",
      "processing song number 162\n",
      "processing song number 163\n",
      "processing song number 164\n",
      "processing song number 165\n",
      "processing song number 166\n",
      "processing song number 167\n",
      "processing song number 168\n",
      "processing song number 169\n",
      "processing song number 170\n",
      "processing song number 171\n",
      "processing song number 172\n",
      "processing song number 173\n",
      "processing song number 174\n",
      "processing song number 175\n",
      "processing song number 176\n",
      "processing song number 177\n",
      "processing song number 178\n",
      "processing song number 179\n",
      "processing song number 180\n",
      "processing song number 181\n",
      "processing song number 182\n",
      "processing song number 183\n",
      "processing song number 184\n",
      "processing song number 185\n",
      "processing song number 186\n",
      "Completed tokenising all song\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combi_to_int = dict((combi, number) for number, combi in enumerate(vocab))\n",
    "int_to_combi = dict((number, combi) for number, combi in enumerate(vocab))\n",
    "\n",
    "all_song_tokenised = []\n",
    "for idx, song in enumerate(all_songs):\n",
    "    print(f\"processing song number {idx}\")\n",
    "    song = mlb.inverse_transform(song)\n",
    "    song = [combi_to_int[tup] if tup in vocab else unk_tag_idx for tup in song]\n",
    "#     song = [combi_to_int[tup] for tup in song]\n",
    "    all_song_tokenised.append(np.array(song))\n",
    "print(f\"Completed tokenising all song\")\n",
    "\n",
    "#delete to free up memory\n",
    "del all_songs\n",
    "del all_songs_np\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import variables from pickle ( if you did not manually process and tokenise data yourself)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# with open('./16v2/combi_to_int.pickle', 'wb') as f:\n",
    "#     pickle.dump(combi_to_int, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "# with open('./16v2/int_to_combi.pickle', 'wb') as f:\n",
    "#     pickle.dump(int_to_combi, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "# with open('./16v2/vocab.pickle', 'wb') as f:\n",
    "#     pickle.dump(vocab, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "# with open('./16v2/all_song_tokenised.pickle', 'wb') as f:\n",
    "#     pickle.dump(all_song_tokenised, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "\n",
    "with open('./16v2/combi_to_int.pickle', 'rb') as f:\n",
    "    combi_to_int = pickle.load(f)\n",
    "    \n",
    "with open('./16v2/all_song_tokenised.pickle', 'rb') as f:\n",
    "    all_song_tokenised = pickle.load(f)\n",
    "\n",
    "with open('./16v2/int_to_combi.pickle', 'rb') as f:\n",
    "    int_to_combi = pickle.load(f)\n",
    "    \n",
    "with open('./16v2/vocab.pickle', 'rb') as f:\n",
    "    vocab = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample data:[ 7018  7018  7018 14629 14629  1797    58    58    58  3388 14628 14628\n",
      " 14628 14628 14628 27783  7084  7084  7084 18104 18104   208   208 11657\n",
      " 14629 14629 14629 14629 14629     2    58    58    58 12162 22881 22881\n",
      " 22881 22881 22881 14628  9183    37    41    41    41  2604  7018  7018\n",
      "  7018  7018  1797  1797    41     2     2     2     2     2 14488 14488\n",
      " 14488 14488 14488 14488 14488 14488     2     2     2     2 22914 22914\n",
      " 22914 22914 22914 22914 22914 22914 22914     2     2     2     2 29682\n",
      " 29682 29682 29682 29682 29682 29682 29682 38982 14618 14618 14618 23162\n",
      " 23162 23162  8399 22088  8399  8399  3483  3483  7258  7258  7258  1875\n",
      "  2051  2051  7258  7258 10363 10363   443   443 24195 24195 24195 24195\n",
      " 24195 24195 35145 16078  7920  7920  7920 14462 14462 14462 14462 23029\n",
      " 14462 14462 14462 17670 17670 17670   104     2     2 12314    24    24\n",
      "    24 28674 14365 14365 14365 14365 14365 14365 14365 14365 24189 24189\n",
      " 20112 20112  4044  4044  4044 24189 14422 14422 24189 24189 24189 16101\n",
      " 16101 16101     0  4017  4017  4017   556   453    50   453   453 16101\n",
      " 16101  2889  2889  2889  2889 21704  6977  7407  7407  7407  7407  6960\n",
      "  6960  6960  6960  6960  6960  6960 14376 14376 18656 18656 18656    35\n",
      "    35    35    35 16556 16556 16556 16556 16556 16556 15061 15061 15061\n",
      " 14750    27     2     2     2     2 27304   636     2     2 15135   133\n",
      "   133   133   133   133 15671 15671 15671 15671 15671 15671 15671 15671\n",
      " 15671    41    41    41    41    41    41   104   104   104   104 22533\n",
      " 16463 16463 30709 16417 16417 16417 14610  1817  1817  1817 10386 10386\n",
      "    41    41    41    41    41    41  7237  1877  1877  1877 22533 22533\n",
      " 22533 22533 22533 22533 29235 29235 23082 23082 23082 23082 23082 30709\n",
      " 30709 30709    41    41    41    41    41    41  1803  1803  1803  1803\n",
      "  2939  2939  2939  2939  8999  8999  8999 24245 23082 23082 23082 23082\n",
      " 23082 23082    41    41   443    41    41   258  7237  7237 14617  7237\n",
      "    13     2  8745  8745  8745  8745 15674 15674 15674 15674 15674 15674\n",
      " 15674 15674 15674 25357  3781  3781  3781  3781 12791 35936 35936 35936\n",
      " 35936 35936 35936 23879 16733 16733 16733 16733 16733 35950 35950 35950\n",
      " 35950 35950 35950   454   454   454    48 27745 18642 18642 18642 24981\n",
      " 24981 24981 24981 14440 14440 14440 14440 14440 14440 28941  8399 17032\n",
      " 17032 17032 17032 17032  8137  4844  4844 12388  7264  7264  7264  7264\n",
      "  7264  7264  7264  7264 14453  7231  7231  7231  7231  7231 14931 14931\n",
      " 14931 14931 14931 14931  9521    15  9521  9521    24    24  7631  7631\n",
      "  7631  7631 16101 16101 16640 16640 16640 16640 24268 24268 24268 24268\n",
      " 24268  1884    58    58  8294  8294  8294  8294  8294  8294  8294     2\n",
      "     2     2     2     2     2 23998 23998 23998 23998 23998 23998 36686\n",
      "  6940  6940  6940 22170  6948  6948  6948  6948  6948  6948  6948  6948\n",
      "  6948  6948  6948  6948 18488 18488 18488    66    66    66    66 15925\n",
      " 15925 15925 15925 15925 15925 15925 15925    65    65    65    65  1196\n",
      "     2     2     2     2     2     2     2    58     2     2     2    26\n",
      "    26  7012  7012  7012 16417 16417 16417  7546  7546  7546  7546  7546\n",
      "  7546 20545  8521  8521  8521  8521  8521 20677   620   620   620   620\n",
      "   620   620 16413 16413 16413 16413 16413 16413 10295 15764 15764 15764\n",
      " 15764 15764 15764 15764 15764 15764 26023 26023 26023 18700 18700 18700\n",
      " 25974 25974 25974 33726 33726 33726 33726 33726 33726 18326 10219 10219\n",
      " 10219 18326 18326 27145 27145   620     2    42    42 15731 15731  7193\n",
      "  7193   878    57  1352  7245  7245  7245  7245  7245  7245 22489    18\n",
      "    18    18   268   268   268   268   268   268  3181  3181     0  1859]\n"
     ]
    }
   ],
   "source": [
    "print(f\"Sample data:{all_song_tokenised[1][:sequence_length]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OyE_rYnJqUjo"
   },
   "source": [
    "## Transformer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZZOK3emyh0D0"
   },
   "source": [
    "### Embedding Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "17Qdsb5Ih4Vk"
   },
   "outputs": [],
   "source": [
    "class TokenAndPositionEmbedding(layers.Layer):\n",
    "    def __init__(self, maxlen, vocab_size, embed_dim):\n",
    "        super(TokenAndPositionEmbedding, self).__init__()\n",
    "        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n",
    "#         self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embed_dim = embed_dim\n",
    "        self.maxlen = maxlen\n",
    "        self.maximum_position_encoding = 10000\n",
    "        \n",
    "    def get_config(self):\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'vocab_size': self.vocab_size,\n",
    "            'embed_dim': self.embed_dim,\n",
    "            'maxlen': self.maxlen,\n",
    "        })\n",
    "        return config\n",
    "    \n",
    "    def get_angles(self, pos, i, d_model):\n",
    "        angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
    "        return pos * angle_rates\n",
    "    \n",
    "    def positional_encoding(self, position, d_model):\n",
    "        angle_rads = self.get_angles(np.arange(position)[:, np.newaxis],\n",
    "                              np.arange(d_model)[np.newaxis, :],\n",
    "                              d_model)\n",
    "\n",
    "        # apply sin to even indices in the array; 2i\n",
    "        angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "        # apply cos to odd indices in the array; 2i+1\n",
    "        angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "        pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "        return tf.cast(pos_encoding, dtype=tf.float32)\n",
    "        \n",
    "    def call(self, x):\n",
    "        maxlen = tf.shape(x)[-1]\n",
    "        pos_encoding = self.positional_encoding(self.maximum_position_encoding, self.embed_dim)\n",
    "        x = self.token_emb(x)\n",
    "        return x + pos_encoding[:,:maxlen,:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# maxlen=5\n",
    "# vocab_size=12\n",
    "# embed_dim = 3\n",
    "# b= np.zeros((3,5))\n",
    "# bb = tf.convert_to_tensor(b, dtype=float)\n",
    "\n",
    "# # inputs = layers.Input(shape=(maxlen,), dtype=tf.int32)\n",
    "# embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n",
    "# # x = embedding_layer(inputs)\n",
    "# x = embedding_layer(bb)\n",
    "# print(x )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gh3RhUvE8cKt"
   },
   "source": [
    "### Self-attention with causal masking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "lfaLa27o9AgN"
   },
   "outputs": [],
   "source": [
    "class MultiHeadSelfAttention(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads=8):\n",
    "        #defining no of nodes/dim for each layer\n",
    "        super(MultiHeadSelfAttention, self).__init__()\n",
    "        self.embed_dim = embed_dim\n",
    "        self.num_heads = num_heads\n",
    "        if embed_dim % num_heads != 0:\n",
    "            raise ValueError(\n",
    "                f\"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}\"\n",
    "            )\n",
    "        self.projection_dim = embed_dim // num_heads\n",
    "        self.query_dense = layers.Dense(embed_dim)\n",
    "        self.key_dense = layers.Dense(embed_dim)\n",
    "        self.value_dense = layers.Dense(embed_dim)\n",
    "        self.combine_heads = layers.Dense(embed_dim)\n",
    "        \n",
    "    def get_config(self):\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'embed_dim': self.embed_dim,\n",
    "            'num_heads': self.num_heads,\n",
    "        })\n",
    "        return config\n",
    "\n",
    "    @staticmethod\n",
    "    def causal_attention_mask(n_dest, n_src, dtype):\n",
    "        \"\"\"\n",
    "        1's in the lower triangle, counting from the lower right corner.\n",
    "        \"\"\"\n",
    "        i = tf.range(n_dest)[:, None]\n",
    "        j = tf.range(n_src)\n",
    "        m = i >= j - n_src + n_dest\n",
    "        return tf.cast(m, dtype)\n",
    "\n",
    "    def attention(self, query, key, value):\n",
    "        score = tf.matmul(query, key, transpose_b=True)\n",
    "        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n",
    "        scaled_score = score / tf.math.sqrt(dim_key)\n",
    "\n",
    "        # prevent information flow from future tokens\n",
    "        shape = tf.shape(scaled_score)\n",
    "        dim_dest, dim_src = shape[2], shape[3]\n",
    "        attention_mask = self.causal_attention_mask(\n",
    "            dim_dest, dim_src, scaled_score.dtype\n",
    "        )\n",
    "        attention_mask = tf.reshape(attention_mask, [1, 1, dim_dest, dim_src])\n",
    "        scaled_score = scaled_score * attention_mask - 1e4 * (1 - attention_mask)\n",
    "\n",
    "        weights = tf.nn.softmax(scaled_score, axis=-1)\n",
    "        output = tf.matmul(weights, value)\n",
    "        return output, weights\n",
    "\n",
    "    def separate_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "\n",
    "    def call(self, inputs):\n",
    "        # x.shape = [batch_size, seq_len, embedding_dim]\n",
    "        batch_size = tf.shape(inputs)[0]\n",
    "        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        key = self.key_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)\n",
    "        query = self.separate_heads(\n",
    "            query, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        key = self.separate_heads(\n",
    "            key, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        value = self.separate_heads(\n",
    "            value, batch_size\n",
    "        )  # (batch_size, num_heads, seq_len, projection_dim)\n",
    "        attention, weights = self.attention(query, key, value)\n",
    "        attention = tf.transpose(\n",
    "            attention, perm=[0, 2, 1, 3]\n",
    "        )  # (batch_size, seq_len, num_heads, projection_dim)\n",
    "        concat_attention = tf.reshape(\n",
    "            attention, (batch_size, -1, self.embed_dim)\n",
    "        )  # (batch_size, seq_len, embed_dim)\n",
    "        output = self.combine_heads(\n",
    "            concat_attention\n",
    "        )  # (batch_size, seq_len, embed_dim)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vNXWdP8DgnUK"
   },
   "source": [
    "### Transformer block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "CDvR-OPdgqbu"
   },
   "outputs": [],
   "source": [
    "class TransformerBlock(layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, dropout_rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "        self.embed_dim = embed_dim\n",
    "        self.num_heads = num_heads\n",
    "        self.ff_dim = ff_dim\n",
    "        self.dropout_rate = dropout_rate\n",
    "        \n",
    "        self.att = MultiHeadSelfAttention(embed_dim, num_heads)\n",
    "        self.ffn = keras.Sequential(\n",
    "            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n",
    "        )\n",
    "        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = layers.Dropout(dropout_rate)\n",
    "        self.dropout2 = layers.Dropout(dropout_rate)\n",
    "        \n",
    "    def get_config(self):\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'embed_dim': self.embed_dim,\n",
    "            'num_heads': self.num_heads,\n",
    "            'ff_dim': self.ff_dim,\n",
    "            'dropout_rate': self.dropout_rate,\n",
    "        })\n",
    "        return config\n",
    "\n",
    "    def call(self, inputs):\n",
    "        attention_output = self.att(inputs)\n",
    "        attention_output = self.dropout1(attention_output)\n",
    "        out1 = self.layernorm1(inputs + attention_output)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output)\n",
    "        return self.layernorm2(out1 + ffn_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_wH-kaFSjC7G"
   },
   "source": [
    "### The Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "0I0tMJJBjGNd"
   },
   "outputs": [],
   "source": [
    "train_loss = []\n",
    "val_loss = []\n",
    "\n",
    "def create_model():\n",
    "    inputs = layers.Input(shape=(maxlen,), dtype=tf.int32)\n",
    "    embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n",
    "    x = embedding_layer(inputs)\n",
    "    transformer_block1 = TransformerBlock(embed_dim, num_heads, feed_forward_dim, dropout_rate = 0.25)\n",
    "    transformer_block2 = TransformerBlock(embed_dim, num_heads, feed_forward_dim, dropout_rate = 0.25)\n",
    "    transformer_block3 = TransformerBlock(embed_dim, num_heads, feed_forward_dim, dropout_rate = 0.25)\n",
    "    x = transformer_block1(x)\n",
    "    x = transformer_block2(x)\n",
    "    x = transformer_block3(x)\n",
    "    outputs = layers.Dense(vocab_size)(x)\n",
    "    model = keras.Model(inputs=inputs, outputs=[outputs, x])\n",
    "    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "    opt = keras.optimizers.Adam(learning_rate=0.001)\n",
    "    model.compile(\n",
    "        \"adam\", loss=[loss_fn, None],\n",
    "    )  # No loss and optimization based on word embeddings from transformer block\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "600"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 600)]             0         \n",
      "_________________________________________________________________\n",
      "token_and_position_embedding (None, 600, 128)          5120000   \n",
      "_________________________________________________________________\n",
      "transformer_block (Transform (None, 600, 128)          99584     \n",
      "_________________________________________________________________\n",
      "transformer_block_1 (Transfo (None, 600, 128)          99584     \n",
      "_________________________________________________________________\n",
      "transformer_block_2 (Transfo (None, 600, 128)          99584     \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 600, 40000)        5160000   \n",
      "=================================================================\n",
      "Total params: 10,578,752\n",
      "Trainable params: 10,578,752\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = create_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.save('./MuGenTransformer-v3-data16v2-stable')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator\n",
    "Custom generator to input one random sequence from each song to train. (Instead of the old method of one shot loading all iterative sequence to the model to train, referenced from MusicTransformer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class My_Custom_Generator(keras.utils.Sequence) :\n",
    "    def __init__(self, all_song_tokenised, batch_size, sequence_length, val_split = 0, shuffle=True) :\n",
    "        self.all_song_tokenised = all_song_tokenised\n",
    "        self.pad_tag_idx = 1\n",
    "        self.sequence_length = sequence_length\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.val_split = val_split\n",
    "        if(self.val_split != 0):\n",
    "            self.all_song_tokenised = random.choices(self.all_song_tokenised, k = int(self.val_split*len(self.all_song_tokenised)))\n",
    "            self.batch_size = len(self.all_song_tokenised)\n",
    "        self.on_epoch_end()\n",
    "    \n",
    "    def __len__(self) :\n",
    "#         return (np.ceil((len(self.pickle_filenames)* self.data_per_file)/ float(self.batch_size))).astype(np.int)\n",
    "        return int(np.ceil(len(self.all_song_tokenised)/ self.batch_size))\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        'Updates indexes after each epoch'\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.all_song_tokenised)\n",
    "  \n",
    "    def __getitem__(self, idx) :\n",
    "        batch_x = np.empty((0, self.sequence_length), float)\n",
    "        batch_y = np.empty((0, self.sequence_length), float)\n",
    "        for i in range(self.batch_size):\n",
    "            if(idx*self.batch_size + i == len(self.all_song_tokenised)-1):\n",
    "                return batch_x, batch_y\n",
    "            song = self.all_song_tokenised[idx*self.batch_size + i]\n",
    "            start_idx = random.randint(0,len(song) - self.sequence_length/2)\n",
    "            seq = song[start_idx: start_idx + self.sequence_length + 1]\n",
    "            x= seq[:-1]\n",
    "            y = seq[1:]\n",
    "#           padding if needed\n",
    "            if(len(y) < self.sequence_length):\n",
    "                no_of_pad = self.sequence_length - len(y)\n",
    "                x = np.append(x, [self.pad_tag_idx]*no_of_pad, axis = 0)\n",
    "                y = np.append(y, [self.pad_tag_idx]*no_of_pad, axis = 0)\n",
    "#             print(idx*batch_size + i)\n",
    "#             while (np.unique(seq).shape[0] == 1):\n",
    "#                 start_idx = random.randint(0,len(song) - self.sequence_length-2)\n",
    "#                 seq = song[start_idx: start_idx + self.sequence_length + 1]\n",
    "            \n",
    "            batch_x = np.append(batch_x, [x], axis = 0)\n",
    "            batch_y = np.append(batch_y, [y], axis = 0)\n",
    "            \n",
    "        return batch_x, batch_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence Generator callback:\n",
    "To show an instance of how the model behave once every specified epochs. Fixed seed sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class GeneratorCallback(keras.callbacks.Callback):\n",
    "    \"\"\"Callback to generate text from trained model.\n",
    "    1. Feed some starting prompt to the model\n",
    "    2. Predict probabilities for next token\n",
    "    3. Sample next token and add it to the next input\n",
    "\n",
    "    # Arguments\n",
    "        max_tokens: Integer, the number of tokens to be generated after prompt.\n",
    "        start_tokens: List of integers, the token indices for the starting prompt.\n",
    "        index_to_word: List of strings, obtained from TextVectorization layer.\n",
    "        top_k: Integer, sample from the `top_k` token predictions.\n",
    "        print_every: Integer, print after this many epochs.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, max_tokens, start_tokens, top_k=10, print_every=5\n",
    "    ):\n",
    "        self.max_tokens = max_tokens\n",
    "        self.start_tokens = start_tokens\n",
    "#         self.index_to_word = index_to_word\n",
    "        self.print_every = print_every\n",
    "        self.k = top_k\n",
    "\n",
    "    def sample_from(self, logits):\n",
    "        logits, indices = tf.math.top_k(logits, k=self.k, sorted=True)\n",
    "        indices = np.asarray(indices).astype(\"int32\")\n",
    "        preds = keras.activations.softmax(tf.expand_dims(logits, 0))[0]\n",
    "        preds = np.asarray(preds).astype(\"float32\")\n",
    "        return np.random.choice(indices, p=preds)\n",
    "\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        start_tokens = [_ for _ in self.start_tokens]\n",
    "        if (epoch + 1) % self.print_every != 0:\n",
    "            return\n",
    "        num_tokens_generated = 0\n",
    "        tokens_generated = []\n",
    "        while num_tokens_generated <= self.max_tokens:\n",
    "            x = start_tokens[-sequence_length:]\n",
    "            pad_len = maxlen - len(start_tokens)\n",
    "            sample_index = -1\n",
    "            if pad_len > 0:\n",
    "                x = start_tokens + [0] * pad_len\n",
    "                sample_index = len(start_tokens) - 1\n",
    "            \n",
    "            x = np.array([x])\n",
    "            y, _ = self.model.predict(x)\n",
    "            sample_token = self.sample_from(y[0][sample_index])\n",
    "            tokens_generated.append(sample_token)\n",
    "            start_tokens.append(sample_token)\n",
    "            num_tokens_generated = len(tokens_generated)\n",
    "            \n",
    "#         txt = \" \".join(\n",
    "#             [self.detokenize(_) for _ in self.start_tokens + tokens_generated]\n",
    "#         )\n",
    "\n",
    "        print(f\"last 40 tokens of starting token:\\n{self.start_tokens[-50:]}\\n\")\n",
    "        print(f\"generated token:\\n{tokens_generated}\\n\")\n",
    "\n",
    "start_tokens = all_song_tokenised[1][:sequence_length-200]\n",
    "num_tokens_generated = 80\n",
    "gen_callback = GeneratorCallback(num_tokens_generated, start_tokens, print_every= generate_sample_every_ep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1500\n",
    "batchsize = 32\n",
    "output_path = f\"./output/MuGenTransformer_v3_{epochs}{batchsize}{int(time.time())}_16v2f/\"\n",
    "\n",
    "\n",
    "my_training_batch_generator = My_Custom_Generator(all_song_tokenised, batchsize, sequence_length)\n",
    "my_validation_batch_generator = My_Custom_Generator(all_song_tokenised, batchsize, sequence_length, val_split=0.1)\n",
    "\n",
    "\n",
    "if not os.path.exists(output_path):\n",
    "    os.mkdir(output_path)\n",
    "\n",
    "# model.load_weights(\"./output/train_multilabel_v3_1_4001000_16th/music-gen-weight.hdf5\")\n",
    "\n",
    "weight_path = output_path + \"music-gen-weight.hdf5\"\n",
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    weight_path,\n",
    "    monitor='loss',\n",
    "    verbose=0,\n",
    "    save_best_only=True,\n",
    "    mode='min'\n",
    ")\n",
    "callbacks_list = [checkpoint,gen_callback]\n",
    "# history = model.fit(network_input,\n",
    "#                     network_output, \n",
    "#                     epochs=epochs, \n",
    "#                     batch_size=batchsize, \n",
    "#                     callbacks=callbacks_list,\n",
    "#                     validation_split=0.1,\n",
    "#                    shuffle=True)\n",
    "\n",
    "history = model.fit(x = my_training_batch_generator,\n",
    "                    callbacks = callbacks_list,                    \n",
    "                   epochs = epochs,\n",
    "                   verbose = 1,\n",
    "                   validation_data = my_validation_batch_generator)\n",
    "\n",
    "train_loss += history.history['loss']\n",
    "val_loss += history.history['val_loss']\n",
    "\n",
    "plt.plot(train_loss)\n",
    "plt.plot(val_loss)\n",
    "plt.title('model train vs validation loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train_loss', 'validation_loss'], loc='upper right')\n",
    "plt.savefig(output_path + 'loss.png')\n",
    "plt.show()\n",
    "print(\"Result stored in {}\".format(output_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference\n",
    "Get a random seq from a random song from input. \n",
    "Then predict from it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generated 1 notes\n",
      "generated 2 notes\n",
      "generated 3 notes\n",
      "generated 4 notes\n",
      "generated 5 notes\n",
      "generated 6 notes\n",
      "generated 7 notes\n",
      "generated 8 notes\n",
      "generated 9 notes\n",
      "generated 10 notes\n",
      "generated 11 notes\n",
      "generated 12 notes\n",
      "generated 13 notes\n",
      "generated 14 notes\n",
      "generated 15 notes\n",
      "generated 16 notes\n",
      "generated 17 notes\n",
      "generated 18 notes\n",
      "generated 19 notes\n",
      "generated 20 notes\n",
      "generated 21 notes\n",
      "generated 22 notes\n",
      "generated 23 notes\n",
      "generated 24 notes\n",
      "generated 25 notes\n",
      "generated 26 notes\n",
      "generated 27 notes\n",
      "generated 28 notes\n",
      "generated 29 notes\n",
      "generated 30 notes\n",
      "generated 31 notes\n",
      "generated 32 notes\n",
      "generated 33 notes\n",
      "generated 34 notes\n",
      "generated 35 notes\n",
      "generated 36 notes\n",
      "generated 37 notes\n",
      "generated 38 notes\n",
      "generated 39 notes\n",
      "generated 40 notes\n",
      "generated 41 notes\n",
      "generated 42 notes\n",
      "generated 43 notes\n",
      "generated 44 notes\n",
      "generated 45 notes\n",
      "generated 46 notes\n",
      "generated 47 notes\n",
      "generated 48 notes\n",
      "generated 49 notes\n",
      "generated 50 notes\n",
      "generated 51 notes\n",
      "generated 52 notes\n",
      "generated 53 notes\n",
      "generated 54 notes\n",
      "generated 55 notes\n",
      "generated 56 notes\n",
      "generated 57 notes\n",
      "generated 58 notes\n",
      "generated 59 notes\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-be61d9551db0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m     \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m     \u001b[0msample_token\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msample_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0mtokens_generated\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_token\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m       raise ValueError('{} is not supported in multi-worker mode.'.format(\n\u001b[1;32m     87\u001b[0m           method.__name__))\n\u001b[0;32m---> 88\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m   return tf_decorator.make_decorator(\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1247\u001b[0m           \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1248\u001b[0m           \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1249\u001b[0;31m           model=self)\n\u001b[0m\u001b[1;32m   1250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1251\u001b[0m       \u001b[0;31m# Container that configures and calls `tf.keras.Callback`s.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model)\u001b[0m\n\u001b[1;32m   1110\u001b[0m         \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m         \u001b[0mdistribution_strategy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mds_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1112\u001b[0;31m         model=model)\n\u001b[0m\u001b[1;32m   1113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1114\u001b[0m     \u001b[0mstrategy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mds_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0;31m# trigger the next permutation. On the other hand, too many simultaneous\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    327\u001b[0m     \u001b[0;31m# shuffles can contend on a hardware level and degrade all performance.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m     \u001b[0mindices_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindices_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpermutation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprefetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    329\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mslice_batch_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, map_func, num_parallel_calls, deterministic)\u001b[0m\n\u001b[1;32m   1619\u001b[0m     \"\"\"\n\u001b[1;32m   1620\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnum_parallel_calls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1621\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mMapDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreserve_cardinality\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1622\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1623\u001b[0m       return ParallelMapDataset(\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, input_dataset, map_func, use_inter_op_parallelism, preserve_cardinality, use_legacy_function)\u001b[0m\n\u001b[1;32m   3979\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transformation_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3980\u001b[0m         \u001b[0mdataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3981\u001b[0;31m         use_legacy_function=use_legacy_function)\n\u001b[0m\u001b[1;32m   3982\u001b[0m     variant_tensor = gen_dataset_ops.map_dataset(\n\u001b[1;32m   3983\u001b[0m         \u001b[0minput_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variant_tensor\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, func, transformation_name, dataset, input_classes, input_shapes, input_types, input_structure, add_to_graph, use_legacy_function, defun_kwargs)\u001b[0m\n\u001b[1;32m   3219\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mtracking\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresource_tracker_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_tracker\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3220\u001b[0m         \u001b[0;31m# TODO(b/141462134): Switch to using garbage collection.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3221\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwrapper_fn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_concrete_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3223\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0madd_to_graph\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mget_concrete_function\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2530\u001b[0m     \"\"\"\n\u001b[1;32m   2531\u001b[0m     graph_function = self._get_concrete_function_garbage_collected(\n\u001b[0;32m-> 2532\u001b[0;31m         *args, **kwargs)\n\u001b[0m\u001b[1;32m   2533\u001b[0m     \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_garbage_collector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2534\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_get_concrete_function_garbage_collected\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2494\u001b[0m       \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2495\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2496\u001b[0;31m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2497\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_signature\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2498\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_signature\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_maybe_define_function\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   2775\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2776\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmissed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcall_context_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2777\u001b[0;31m       \u001b[0mgraph_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_graph_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2778\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprimary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2779\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_create_graph_function\u001b[0;34m(self, args, kwargs, override_flat_arg_shapes)\u001b[0m\n\u001b[1;32m   2665\u001b[0m             \u001b[0marg_names\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0marg_names\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2666\u001b[0m             \u001b[0moverride_flat_arg_shapes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverride_flat_arg_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2667\u001b[0;31m             capture_by_value=self._capture_by_value),\n\u001b[0m\u001b[1;32m   2668\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_attributes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2669\u001b[0m         \u001b[0;31m# Tell the ConcreteFunction to clean up its graph once it goes out of\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36mfunc_graph_from_py_func\u001b[0;34m(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)\u001b[0m\n\u001b[1;32m    979\u001b[0m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moriginal_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munwrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpython_func\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 981\u001b[0;31m       \u001b[0mfunc_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpython_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mfunc_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfunc_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    982\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    983\u001b[0m       \u001b[0;31m# invariant: `func_outputs` contains only Tensors, CompositeTensors,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36mwrapper_fn\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m   3212\u001b[0m           attributes=defun_kwargs)\n\u001b[1;32m   3213\u001b[0m       \u001b[0;32mdef\u001b[0m \u001b[0mwrapper_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=missing-docstring\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3214\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_wrapper_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3215\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstructure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_tensor_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output_structure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3216\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36m_wrapper_helper\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m   3154\u001b[0m         \u001b[0mnested_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnested_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3156\u001b[0;31m       \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mautograph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtf_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mag_ctx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnested_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3157\u001b[0m       \u001b[0;31m# If `func` returns a list of tensors, `nest.flatten()` and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3158\u001b[0m       \u001b[0;31m# `ops.convert_to_tensor()` would conspire to attempt to stack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    260\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mconversion_ctx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m           \u001b[0;32mreturn\u001b[0m \u001b[0mconverted_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    263\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint:disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ag_error_metadata'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mconverted_call\u001b[0;34m(f, args, kwargs, caller_fn_scope, options)\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_requested\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconversion\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_whitelisted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 492\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_call_unconverted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m   \u001b[0;31m# internal_convert_user_code is for example turned off when issuing a dynamic\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36m_call_unconverted\u001b[0;34m(f, args, kwargs, options, update_cache)\u001b[0m\n\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/data_adapter.py\u001b[0m in \u001b[0;36mpermutation\u001b[0;34m(_)\u001b[0m\n\u001b[1;32m    317\u001b[0m       \u001b[0;31m# than reusing the same range Tensor. (presumably because of buffer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m       \u001b[0;31m# forwarding.)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m       \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mshuffle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mshuffle\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"batch\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_shuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mrange\u001b[0;34m(start, limit, delta, dtype, name)\u001b[0m\n\u001b[1;32m   1590\u001b[0m     \u001b[0mdelta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minferred_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1591\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1592\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1593\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1594\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36m_range\u001b[0;34m(start, limit, delta, name)\u001b[0m\n\u001b[1;32m   7122\u001b[0m   \u001b[0;31m# Add nodes to the TensorFlow graph.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7123\u001b[0m   _, _, _op, _outputs = _op_def_library._apply_op_helper(\n\u001b[0;32m-> 7124\u001b[0;31m         \"Range\", start=start, limit=limit, delta=delta, name=name)\n\u001b[0m\u001b[1;32m   7125\u001b[0m   \u001b[0m_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_outputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7126\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0m_execute\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmust_record_gradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\u001b[0m in \u001b[0;36m_apply_op_helper\u001b[0;34m(op_type_name, name, **keywords)\u001b[0m\n\u001b[1;32m    742\u001b[0m       op = g._create_op_internal(op_type_name, inputs, dtypes=None,\n\u001b[1;32m    743\u001b[0m                                  \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_types\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 744\u001b[0;31m                                  attrs=attr_protos, op_def=op_def)\n\u001b[0m\u001b[1;32m    745\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    746\u001b[0m     \u001b[0;31m# `outputs` is returned as a separate return value so that the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[1;32m    593\u001b[0m     return super(FuncGraph, self)._create_op_internal(  # pylint: disable=protected-access\n\u001b[1;32m    594\u001b[0m         \u001b[0mop_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_def\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 595\u001b[0;31m         compute_device)\n\u001b[0m\u001b[1;32m    596\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcapture\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_create_op_internal\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)\u001b[0m\n\u001b[1;32m   3325\u001b[0m           \u001b[0minput_types\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3326\u001b[0m           \u001b[0moriginal_op\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_default_original_op\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3327\u001b[0;31m           op_def=op_def)\n\u001b[0m\u001b[1;32m   3328\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_op_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompute_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompute_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3329\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)\u001b[0m\n\u001b[1;32m   1815\u001b[0m         \u001b[0mop_def\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_op_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1816\u001b[0m       self._c_op = _create_c_op(self._graph, node_def, inputs,\n\u001b[0;32m-> 1817\u001b[0;31m                                 control_input_ops, op_def)\n\u001b[0m\u001b[1;32m   1818\u001b[0m       \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1819\u001b[0m     \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_create_c_op\u001b[0;34m(graph, node_def, inputs, control_inputs, op_def)\u001b[0m\n\u001b[1;32m   1627\u001b[0m   op_desc = pywrap_tf_session.TF_NewOperation(graph._c_graph,\n\u001b[1;32m   1628\u001b[0m                                               \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1629\u001b[0;31m                                               compat.as_str(node_def.name))\n\u001b[0m\u001b[1;32m   1630\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1631\u001b[0m     \u001b[0mpywrap_tf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_SetDevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop_desc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "seed_len = 100\n",
    "num_note_to_gen = 1000\n",
    "\n",
    "song_idx = random.randint(0,len(all_song_tokenised)-1)\n",
    "seq_start_at = random.randint(0,len(all_song_tokenised[song_idx])-sequence_length)   \n",
    "start_tokens = all_song_tokenised[song_idx][seq_start_at:seq_start_at + seed_len].tolist()\n",
    "# start_tokens = inf_song[:seed_len]\n",
    "while (start_tokens == [()]*sequence_length):\n",
    "    print(\"Got all zeros, rerolling\")\n",
    "    song_idx = random.randint(0,len(all_song_tokenised)-1)\n",
    "    seq_start_at = random.randint(0,len(all_song_tokenised[song_idx])-sequence_length)   \n",
    "    start_tokens = all_song_tokenised[song_idx][seq_start_at:seq_start_at + sequence_length].tolist()\n",
    "    \n",
    "ori = start_tokens.copy()\n",
    "backup = ori.copy()\n",
    "# start_tokens = \n",
    "\n",
    "\n",
    "def softmax(x):\n",
    "    \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "    return e_x / e_x.sum(axis=0)\n",
    "\n",
    "\n",
    "def sample_from(logits, k):\n",
    "    logits, indices = tf.math.top_k(logits, k= k, sorted=True)\n",
    "    indices = np.asarray(indices).astype(\"int32\")\n",
    "    preds = np.asarray(logits).astype(\"float32\")\n",
    "    if(unk_tag_idx in indices):\n",
    "        unk_tag_position = np.where(indices == unk_tag_idx)[0].item()\n",
    "        indices = np.delete(indices, unk_tag_position)\n",
    "        preds = np.delete(preds, unk_tag_position)\n",
    "    preds = softmax(preds)\n",
    "#     while out == 0:\n",
    "#         print(f\"predicted <UNK> tag with probability {preds[np.where(indices==0)[0][0]]}\")\n",
    "#         out = np.random.choice(indices, p=preds)\n",
    "    return np.random.choice(indices, p=preds)\n",
    "\n",
    "def convertToRoll(seq_list):\n",
    "#     a = network_input[start_idx].tolist()\n",
    "    seq_list = [int_to_combi[i] for i in seq_list]\n",
    "    roll = mlb.transform(seq_list)\n",
    "    print(seq_list)\n",
    "    return roll\n",
    "\n",
    "\n",
    "k = 10\n",
    "tokens_generated = []\n",
    "num_tokens_generated = 0\n",
    "\n",
    "while num_tokens_generated <= num_note_to_gen:\n",
    "\n",
    "    x = start_tokens[-sequence_length:]\n",
    "    pad_len = maxlen - len(start_tokens)\n",
    "    sample_index = -1\n",
    "    if pad_len > 0:\n",
    "        x = start_tokens + [0] * pad_len\n",
    "        sample_index = len(start_tokens) - 1\n",
    "    \n",
    "    x = np.array([x])\n",
    "    y, _ = model.predict(x)\n",
    "    sample_token = sample_from(y[0][sample_index], 10)\n",
    "    tokens_generated.append(sample_token)\n",
    "    start_tokens.append(sample_token)\n",
    "    num_tokens_generated = len(tokens_generated)\n",
    "    print(f\"generated {num_tokens_generated} notes\")\n",
    "    \n",
    "# print(f\"Piano int seq generated\")\n",
    "piano_roll = convertToRoll(start_tokens)\n",
    "print(\"-------------------------------------------\")\n",
    "ori = convertToRoll(ori)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def piano_roll_to_pretty_midi(piano_roll_in, fs, program=0, velocity = 64):\n",
    "    '''Convert a Piano Roll array into a PrettyMidi object\n",
    "     with a single instrument.\n",
    "    Parameters\n",
    "    ----------\n",
    "    piano_roll : np.ndarray, shape=(128,frames), dtype=int\n",
    "        Piano roll of one instrument\n",
    "    fs : int\n",
    "        Sampling frequency of the columns, i.e. each column is spaced apart\n",
    "        by ``1./fs`` seconds.\n",
    "    program : int\n",
    "        The program number of the instrument.\n",
    "    Returns\n",
    "    -------\n",
    "    midi_object : pretty_midi.PrettyMIDI\n",
    "        A pretty_midi.PrettyMIDI class instance describing\n",
    "        the piano roll.\n",
    "    '''\n",
    "    piano_roll = np.where(piano_roll_in == 1, 64, 0)\n",
    "    notes, frames = piano_roll.shape\n",
    "    pm = pretty_midi.PrettyMIDI(initial_tempo=100.0)\n",
    "    instrument = pretty_midi.Instrument(program=program)\n",
    "\n",
    "    # pad 1 column of zeros so we can acknowledge inital and ending events\n",
    "    piano_roll = np.pad(piano_roll, [(0, 0), (1, 1)], 'constant')\n",
    "    print(piano_roll.shape)\n",
    "    \n",
    "    # use changes in velocities to find note on / note off events\n",
    "    velocity_changes = np.nonzero(np.diff(piano_roll).T)\n",
    "\n",
    "    # keep track on velocities and note on times\n",
    "    prev_velocities = np.zeros(notes, dtype=int)\n",
    "    note_on_time = np.zeros(notes)\n",
    "\n",
    "    for time, note in zip(*velocity_changes):\n",
    "        # use time + 1 because of padding above\n",
    "        velocity = piano_roll[note, time + 1]\n",
    "        time = time / fs\n",
    "        if velocity > 0:\n",
    "            if prev_velocities[note] == 0:\n",
    "                note_on_time[note] = time\n",
    "                prev_velocities[note] = velocity\n",
    "        else:\n",
    "            pm_note = pretty_midi.Note(\n",
    "                velocity=prev_velocities[note],\n",
    "                pitch=note,\n",
    "                start=note_on_time[note],\n",
    "                end=time)\n",
    "            instrument.notes.append(pm_note)\n",
    "            prev_velocities[note] = 0\n",
    "    pm.instruments.append(instrument)\n",
    "    return pm\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export as MIDI\n",
    "Save the inference result to output folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bpm = 150\n",
    "fs = 1/((60/bpm)/4)\n",
    "name = \"random2\"\n",
    "# fs = 100\n",
    "# ori = np.array(ori)\n",
    "mid_out = piano_roll_to_pretty_midi(piano_roll.T, fs=fs)\n",
    "mid_ori = piano_roll_to_pretty_midi(ori.T, fs=fs)\n",
    "midi_out_path = output_path+f\"gpt-v3-id-{name}.mid\"\n",
    "if midi_out_path is not None:\n",
    "        mid_out.write(midi_out_path)\n",
    "        \n",
    "midi_ori_path = output_path+f\"ori-gpt-v3-id-{name}.mid\"\n",
    "if midi_ori_path is not None:\n",
    "        mid_ori.write(midi_ori_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save full length of seed song for reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(74,), (74,), (74,), (74,), (55, 86), (55, 86), (55, 86), (55, 86), (55, 58, 86), (55, 58, 86), (55, 58, 86), (55, 58, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 86), (55, 58, 62, 69, 70, 86), (55, 58, 62, 69, 70, 86), (55, 58, 62, 69, 70, 74, 86), (55, 58, 62, 69, 70, 74, 86), (55, 58, 62, 69, 70, 74, 86), (55, 58, 62, 69, 70, 74, 86), (54, 55, 58, 62, 69, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 74, 86), (54, 55, 58, 62, 73, 74, 86), (54, 55, 58, 62, 73, 74, 86), (54, 55, 58, 62, 73, 74, 86), (54, 55, 58, 62, 73, 74, 86), (54, 55, 58, 62, 73, 74, 86), (53, 54, 55, 62, 86), (53, 54, 55, 62, 86), (53, 54, 55), (53, 54, 55), (53, 54, 55, 58, 86), (53, 54, 55, 58, 86), (53, 54, 55, 58, 86), (53, 54, 55, 58, 86), (53, 54, 55, 58, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86), (53, 54, 55, 58, 62, 86, 91), (53, 54, 55, 58, 62, 86, 91), (53, 54, 55, 58, 62, 86, 91), (53, 54, 55, 58, 62, 86, 91), (53, 54, 55, 58, 62, 86, 91), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), (53, 54, 58, 62, 69, 78, 90), '<UNK>', (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (43, 50, 53, 57, 58, 62), (50, 54, 58, 62), (50, 54, 58, 62), (50, 54, 58, 62), (50, 54, 58, 62), (42, 50, 54, 58, 62), (42, 50, 54, 58, 62), (42, 50, 54, 58, 62), (42, 50, 54, 58, 62), (41, 53, 58, 62), (41, 53, 58, 62), (58, 62), (58, 62), (58, 62), (58, 62), (58, 62), (58, 62), (58, 62), (58, 62, 65), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (53, 55, 58, 62, 65, 67, 70, 74), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (38, 54), (38, 54), (38, 54), (38, 54), (38, 54), (38, 54), (38, 54), (38, 54), (38, 54), (58,), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 58), (43, 50), (43, 50, 57, 58), (43, 50, 58, 62), (43, 50), (43, 50), (43, 50), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54,), (54, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54,), (54, 57, 58, 62), (54, 62), (54,), (54,), (54, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (54, 57, 58, 62), (50, 57, 58, 62), (50, 57, 58, 62), (50, 57, 58, 62), (50, 57, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 57, 58, 62), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65), (43, 53, 55, 57, 58, 62, 65, 67), (43, 53, 55, 57, 58, 62, 65, 67), (74,), (48, 58, 63, 67, 70, 74), (48, 58), (48, 58), (48, 58, 77), (48, 58, 77), (48, 58, 77), (48, 58, 77), (48, 58, 77), '<UNK>', (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 75), (53, 58, 63, 67, 68, 74, 75), (60, 63, 68, 74), (60, 63, 68, 74), (63, 68, 74), (58, 63, 68, 74), (58, 63, 68, 74), (58, 63, 68, 74), (58, 63, 68, 74), (58, 63, 68, 74), (58, 63, 68, 74), (58, 62, 63, 68, 74), (58, 62, 63, 68, 74), (62, 68), (62, 68), (56, 62, 68), (56, 62, 68), (46,), (), (), (), (39, 60, 65, 70), (39, 65, 70), (27,), (27,), (27,), (27,), (27,), (27,), (27,), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 77, 82), (27, 55, 60, 65, 72, 74, 77, 82), (27, 55, 60, 65, 72, 74, 77, 82), (27, 55, 60, 65, 72, 74, 77, 82), (62, 67, 69, 74), (62, 63, 67, 69, 74), (62, 63, 67, 69, 74), (62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (50, 62, 63, 67, 69, 74), (60, 63, 66), (60, 63, 66), (60, 63, 66, 74), (60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74), (50, 60, 63, 66, 74, 81), (50, 60, 63, 66, 74, 81), (50, 60, 63, 66, 74, 81), (55, 81), (55, 81), (55, 81), (55, 78, 81), (55, 62, 78, 81), (55, 62, 78, 81), (55, 62, 78, 81), (55, 62, 78, 81), (55, 62, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (55, 62, 66, 69, 74, 78, 81), (62,), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (54, 62, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (41, 53, 58, 62, 65, 70), (), (48, 55, 58, 63, 67), (48, 55, 58, 63, 67), (48, 55, 58, 63, 67), (48, 55, 58, 63, 67), (48, 55, 58, 63, 67, 75, 79), (48, 55, 58, 63, 67, 75, 79), (48, 55, 58, 63, 67, 75, 79), (48, 55, 58, 63, 67, 75, 79), (48, 55, 58, 63, 67, 75, 79), (63, 65, 69, 74, 77), (63, 65, 69, 74, 77), (63, 65, 69, 74, 77), (63, 65, 69, 74, 77), (63, 65, 69, 74, 77), (57, 63, 65, 69, 74, 77), (57, 63, 65, 69, 74, 77), (57, 63, 65, 69, 74, 75, 77), (57, 63, 65, 69, 74, 75, 77), (57, 63, 65, 69, 74, 75, 77), (46, 56, 68, 74), (46, 56, 68, 74), (46, 56, 68, 74), (46, 56, 68, 74), (46, 53, 56, 68, 74), (46, 53, 56, 68, 74), (46, 53, 56, 68, 74), (46, 53, 56, 68, 74), (46, 53, 56, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (46, 53, 56, 62, 65, 68, 74), (44,), (), (), (), (), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (54, 58, 60, 65, 74, 82, 86), (44, 54, 58, 60, 65, 74, 82, 86), (44, 54, 58, 60, 65, 74, 82, 86), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (43, 50, 53, 57, 59, 62), (65, 71, 74, 77, 80), (65, 71, 74, 77, 80), (65, 71, 74, 77, 80), (65, 71, 74, 77, 80), (), (63, 71, 74, 79), (63, 71, 74, 79), (63, 71, 74, 79), (63, 71, 74, 79), (), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (59, 62, 68, 71, 74, 77), (59, 62, 68, 71, 74, 77), (59, 62, 68, 71, 74, 77), (59, 62, 68, 71, 74, 77), (59, 62, 68, 71, 74, 77), (48, 58, 63, 67, 74), (48, 58, 63, 67, 74), (48, 58, 63, 67, 74), (48, 58, 59, 63, 67, 74), (48, 58, 59, 63, 67, 74), (48, 58, 59, 63, 67, 74), (48, 58, 59, 60, 63, 67, 74), (48, 58, 59, 60, 63, 67, 74), (48, 58, 59, 60, 63, 67, 74), (62, 63, 67, 74), (62, 63, 67, 74), (62, 63, 67, 74), (59, 62, 63, 67, 74), (59, 62, 63, 67, 74), (59, 62, 63, 67, 74), (59, 60, 62, 63, 67, 74), (59, 60, 62, 63, 67, 74), (59, 60, 62, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (46, 55, 63, 67, 70, 74), (), (45,), (), (), (), (), (55, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (54, 60, 65, 74, 82, 86), (54, 60, 65, 74, 86), (54, 60, 65, 74, 86), (54, 60, 65, 74, 86), (54, 60, 65, 74, 86), (38, 54, 60, 65, 74, 86), (38, 54, 60, 65, 74, 86), (38, 54, 60, 65, 74, 86), (38, 54, 60, 65, 74, 86), (38, 54, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (38, 50, 54, 58, 60, 65, 74, 86), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (43, 50, 55, 57, 58), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 57, 60, 63), (45, 55, 57, 60, 63), (45, 55, 57, 60, 63), (45, 55, 57, 60, 63), (45, 55, 57, 60, 63, 74), (45, 55, 57, 60, 63, 74), (45, 55, 57, 60, 63, 74), (45, 55, 57, 60, 63, 74), (45, 55, 57, 60, 63, 74), (55, 79), (55, 79), (55, 79), (55, 79), (55, 62, 79), (55, 62, 79), (55, 62, 79), (55, 62, 79), (55, 62, 79), (55, 62, 67, 79), (55, 62, 67, 79), (55, 62, 67, 79), (55, 62, 67, 69, 79), (55, 62, 67, 69, 79), (55, 62, 67, 69, 79), (55, 62, 67, 69, 70, 79), (55, 62, 67, 69, 70, 79), (55, 62, 67, 69, 70, 79), (60, 63, 68, 72), (60, 63, 68, 72), (60, 63, 68, 72), (58, 60, 63, 68, 72, 74), (58, 60, 63, 68, 72, 74), (58, 60, 63, 68, 72, 74), (58, 60, 63, 68, 72, 74, 75), (58, 60, 63, 68, 72, 74, 75), (58, 60, 63, 68, 72, 74, 75, 77), (56, 58, 60, 63, 67, 68, 72, 74, 75, 77), (56, 62, 67, 79), (56, 62, 67, 79), (56, 62, 67, 79, 80), (56, 62, 67, 79, 80), (56, 62, 67, 79, 80, 82), (56, 62, 67, 79, 80, 82, 84), (56, 62, 67, 79, 80, 82, 84, 86), (56, 62, 67, 79, 80, 82, 84, 86), (56, 62, 67, 79, 80, 82, 84, 86, 87), (), (39, 86), (39, 86), (39, 46, 86), (39, 46, 86), (39, 46, 55, 86), (39, 46, 55, 86), (39, 46, 55, 58, 86), (39, 46, 55, 58, 86), (39, 46, 55, 58, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (39, 46, 55, 58, 62, 67, 70, 74, 86), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (45, 55, 60, 63, 67), (34, 46, 57, 60, 62, 65), (46, 60, 65), (46, 60, 65), (46, 60, 65), (46, 60, 65), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 77), (46, 60, 62, 65, 69, 72, 74, 76, 77), (46, 60, 62, 65, 69, 72, 74, 76, 77), (46, 60, 62, 65, 69, 72, 74, 76, 77), (46, 60, 62, 65, 69, 72, 74, 76, 77), (46, 60, 62, 65, 69, 72, 74, 76, 77), (67, 68, 71, 74, 77), (67, 68, 71, 74, 77), (67, 68, 71, 74, 77), (67, 68, 71, 74, 77), (65, 68, 71, 74, 77), (65, 68, 71, 74, 77), (65, 68, 71, 74, 77), (65, 68, 71, 74, 77), (65, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (62, 68, 71, 74, 77), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 58, 63, 67, 75), (48, 58, 63, 67, 75), (48, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 62, 63, 67, 75), (48, 55, 58, 62, 63, 67, 75), (48, 55, 58, 62, 63, 67, 75), (48, 55, 58, 62, 63, 67, 75), (48, 55, 58, 62, 63, 67, 72, 75), (48, 55, 58, 62, 63, 67, 72, 75), (48, 55, 58, 62, 63, 67, 72, 75, 79), (48, 55, 58, 62, 63, 67, 72, 75, 79, 84), (45, 55, 84), (45, 55, 84, 87), (45, 55, 84, 87, 91), (45, 55, 84, 87, 91), (45, 55, 84, 86, 87, 91), (45, 55, 84, 86, 87, 91), (45, 55, 84, 86, 87, 91), (45, 55, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (45, 55, 57, 60, 63, 84, 86, 87, 91), (55, 60, 61, 65, 79, 84, 91), (55, 60, 61, 65, 79, 84, 91), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65, 69), (61, 65, 69), (61, 65, 69), (55, 61, 65, 69), (61, 65, 69), (61,), (39,), (), (), (), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (38, 50, 54, 57, 58, 60), (57, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (56, 57, 58, 62, 66, 69), (55, 56, 57, 58, 62, 66, 69), (54, 55, 56, 57, 58, 62, 66, 69), (54, 55, 56, 57, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (54, 57, 58, 62, 66, 69), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (43, 53, 57, 58, 62, 65), (33, 45, 55, 60, 63), (33, 45, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (33, 55, 60, 63), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (26, 38, 54, 58, 60), (43, 50, 57, 58, 62), (50,), (50,), (50,), (50, 81), (50, 81), (50, 81), (50, 81), (50, 81), (50, 57, 58, 62, 74, 81, 86), (50, 57, 58, 62, 74, 81, 86), (50, 57, 58, 62, 74, 81, 86), (50, 57, 58, 62, 74, 81, 86), (50, 57, 58, 62, 74, 81, 86), (50, 57, 58, 62, 74, 81, 86, 93), (50, 57, 58, 62, 74, 81, 86, 93), (50, 57, 58, 62, 74, 81, 86, 93), (50, 57, 58, 62, 74, 81, 86, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 79, 81, 86, 90, 91, 93), (54, 58, 62, 79, 81, 86, 90, 91, 93), (54, 58, 62, 79, 81, 86, 90, 91, 93), (54, 58, 62, 79, 81, 86, 90, 91, 93), (54, 58, 62, 79, 81, 86, 90, 91, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 86, 90, 93), (54, 58, 62, 81, 82, 86, 90, 93, 94), (54, 58, 62, 81, 82, 86, 90, 93, 94), (54, 58, 62, 81, 82, 86, 90, 93, 94), (54, 58, 62, 81, 82, 86, 90, 93, 94), (54, 58, 62, 81, 82, 86, 90, 93, 94), (53, 58, 62, 81, 86, 93), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 81, 86), (53, 58, 62, 65, 70, 74, 75, 81, 86), (53, 58, 62, 65, 70, 74, 75, 77, 81, 86), (53, 58, 62, 65, 70, 74, 75, 77, 79, 81, 86), '<UNK>', (58, 62, 63, 67, 82), (58, 62, 63, 67, 84), (58, 62, 63, 67, 84, 86), (58, 62, 63, 67, 84, 86), (58, 62, 63, 67, 84, 86), (58, 62, 63, 67, 84, 86, 87), (58, 62, 63, 67, 84, 86, 87), (58, 62, 63, 67, 84, 86, 87, 89), (58, 62, 63, 67, 84, 86, 87, 89), (57, 63, 67, 91), (57, 63, 67, 91), (57, 63, 67, 86, 91), (57, 63, 67, 86, 91), (57, 63, 67, 86, 91), (57, 63, 67, 86, 87, 91), (57, 63, 67, 86, 87, 91), (57, 63, 67, 86, 87, 91), (57, 63, 67, 86, 87, 91), (57, 63, 67, 86, 87, 91), (), (), (), (50, 56, 61), (50, 56, 61), (50, 56, 61), (50, 56, 61), (50, 56, 61), (50, 56, 61), (50, 56, 61, 62, 68, 73), (50, 56, 61, 62, 68, 73), (50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (38, 50, 56, 61, 62, 68, 73), (27, 38, 39, 50, 56, 61, 62, 68, 73), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (27, 39, 55, 60, 65), (), (45, 55, 60, 63), (), (), (), (), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (55, 57, 60, 63, 74, 81, 86), (38, 50), (50,), (50,), (50,), (50,), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (50, 54, 58, 60, 65, 74, 82, 86), (86,), (43, 50, 74, 79, 82, 86), (50, 74, 79, 82, 86), (50, 74, 79, 82, 86), (50, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (50, 57, 58, 62, 74, 79, 82, 86), (54, 58, 62, 74), (54, 58, 62, 74), (54, 74), (54, 74), (54, 58, 62, 74), (54, 58, 62, 74), (54, 58, 62, 74), (54, 58, 62, 74), (54, 58, 62, 74), (54, 58, 62, 74), (54, 58, 62, 66, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (54, 58, 62, 66, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62, 65, 70, 74), (53, 58, 62), (53, 58, 62), (53, 58, 62), (53, 58, 62, 74), (53, 58, 62, 74), (53, 58, 62, 74, 77), (53, 58, 62, 74, 77), (53, 58, 62, 74, 77), (53, 58, 62, 74, 77), (55, 57, 60, 63), (55, 57, 60, 63, 75, 77), (75, 77), (75, 77), (75, 77), (75, 77), (75, 77), (75, 77), (75, 77), (74,), (60, 63, 66, 69, 74), (60, 63, 66, 69), (60, 63, 66, 69), (57, 63, 66, 69, 72), (57, 63, 66, 72), (57, 63, 66, 72), (57, 63, 66, 72), (57, 63, 66, 72), (57, 63, 66, 72), (57, 63, 66, 72), (57, 58, 62, 66, 69), (57, 58, 62, 66, 69), (58, 62, 66, 69), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (55, 58, 62, 66, 67), (54, 55, 58, 62, 66, 67), (54, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), (57, 58, 62, 66), (57, 58, 62, 66), (57, 58, 62, 66), (54, 58, 62, 66), (54, 58, 62, 66), '<UNK>', (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (53, 58, 62, 65), (50, 53, 58, 62, 65), (50, 53, 58, 62, 65), (50, 53, 58, 62, 65), (50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 58, 62, 65), (43, 50, 53, 55, 58, 62, 65), (43, 50, 53, 55, 58, 62, 65), (43, 50, 53, 55, 58, 62, 65), (43, 50, 53, 55, 58, 62, 65, 74), (43, 50, 53, 55, 58, 62, 65, 74), (43, 50, 53, 55, 58, 62, 65, 74, 77), (43, 50, 53, 55, 58, 62, 65, 74, 77), (43, 50, 53, 55, 58, 62, 65, 74, 77), (43, 50, 53, 55, 58, 62, 65, 74, 77), (43, 50, 53, 55, 58, 62, 65, 74, 77), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67, 75), (48, 55, 58, 63, 67), (48, 55, 58, 63, 67, 74), (48, 55, 58, 63, 67, 74), (48, 55, 58, 63, 67, 74), (48, 55, 58, 63, 67, 74), (41, 48, 55, 58, 63, 67, 74), (41, 48, 55, 58, 63, 67, 74), (41, 48, 51, 55, 58, 63, 67, 74), (41, 48, 51, 55, 57, 58, 62, 63, 66, 67, 69, 74), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 72), (51, 57, 62, 66, 69, 70, 72), (51, 57, 62, 66, 69, 70, 72), (51, 57, 62, 66, 69, 70, 72), (73,), (73,), (73,), (56, 61, 70), (50, 56, 61, 70), (50, 56, 61, 72), (50, 56, 61, 72), (50, 56, 61, 72), (50, 56, 61, 65), (50, 56, 61), (50, 56, 61, 70), (50, 56, 61, 70), (50, 56, 61, 70), (50, 56, 61, 70, 72), (50, 56, 61, 70, 72), (50, 56, 61, 70, 72), (50, 56, 61, 70, 72), (50, 56, 61, 70, 72), (39,), (39,), (), (), (), (), (65,), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65), (55, 60, 61, 65, 70), (46, 55, 60, 61, 65, 70, 72), (46, 55, 60, 61, 65, 70, 72), (45, 74), (74,), (74,), (74,), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74), (55, 57, 60, 63, 74, 79), (55, 57, 60, 63, 74, 79), (55, 57, 60, 63, 74, 79, 84), (55, 57, 60, 63, 74, 79, 84, 87), (55, 57, 60, 63, 74, 79, 84, 87), (55, 57, 60, 63, 91), (55, 57, 60, 63, 91), (55, 57, 60, 63, 87, 91), (55, 57, 60, 63, 87, 91), (50, 54, 60, 91), (50, 54, 60, 91), (50, 54, 60, 90, 91), (50, 54, 60, 90, 91), (50, 54, 60, 90, 91), (50, 54, 60, 90, 91), (54, 60, 65, 89), (54, 60, 65, 89), (54, 60, 65, 89), (54, 60, 65, 89), (54, 60, 65, 87, 89), (54, 60, 65, 87, 89), (54, 60, 65, 87, 89), (54, 60, 65, 86, 87, 89), (54, 60, 65, 86, 87, 89), (54, 60, 65, 86, 87, 89), (86, 89), (54, 84, 89), (54, 84, 89), (54, 84, 89), (54, 82, 84, 89), (54, 82, 84, 89), (43, 81), (43, 81), (43, 81), (43, 81), (43, 50, 81), (43, 50, 81), (43, 50, 58, 62, 81), (43, 50, 58, 62, 81), (43, 50, 58, 62, 81), (43, 50, 58, 62, 81), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 55, 58, 62, 67), (43, 50, 55, 58, 62, 67), (43, 50, 58, 62, 67), (43, 50, 58, 62, 67), (43, 50, 55, 58, 62, 67), (43, 50, 55, 58, 62, 67), (43, 50, 55, 58, 62, 66, 67), (43, 50, 55, 58, 62, 66, 67), (43, 50, 55, 58, 62, 66, 67), (43, 50, 55, 58, 62, 66, 67), (43, 50, 55, 58, 62, 66, 67), (43, 50, 55, 58, 62, 66, 67), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 69), (54, 58, 62, 66, 67, 69), (54, 58, 62, 66, 67, 69), (54, 58, 62, 66, 67, 69), (54, 58, 62, 66, 67, 69), (54, 58, 62, 66, 67, 69), (58, 62, 66), (58, 62, 66), (58, 62, 66), (58, 62, 66), (54, 58, 62, 66, 67), (54, 58, 62, 66, 67), (54, 58, 62, 66, 67), (54, 58, 62, 66, 67), (54, 58, 62, 66, 67), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 58, 62), (43, 53, 57, 58, 62), (43, 53, 57, 58, 62), (43, 53, 57, 58, 62), (43, 53, 57, 58, 62), (43, 53, 57, 58, 62, 69), (43, 53, 57, 58, 62, 69, 70), (43, 53, 57, 58, 62, 69, 70), (43, 53, 57, 58, 62, 69, 70, 74), (43, 53, 57, 58, 62, 69, 70, 74), (43, 53, 57, 58, 62, 69, 70, 74, 77), (43, 53, 57, 58, 62, 69, 70, 74, 77, 81), (43, 53, 57, 58, 62, 69, 70, 74, 77, 81), (43, 53, 57, 58, 62, 69, 70, 74, 77, 81, 84), (57, 58, 62, 84), (57, 58, 62, 84), (57, 58, 62, 84), (57, 58, 62, 82, 84), (57, 58, 62, 82, 84), (48, 58, 75, 79), (48, 58, 75, 79), (48, 58, 75, 79), (48, 55, 58, 75, 79), (48, 55, 58, 75, 79), (48, 55, 58, 75, 79), (48, 55, 58, 70, 75, 79), (48, 55, 58, 70, 75, 79), (48, 55, 58, 70, 75, 79), (48, 55, 58, 69, 70, 75, 79), (48, 55, 58, 69, 70, 75, 79), (48, 55, 58, 69, 70, 75, 79), (53, 63, 69, 73, 77), (53, 63, 69, 74, 77), (53, 63, 69), (53, 60, 63, 69, 75), (53, 60, 63, 69, 75), (53, 60, 63, 69, 74, 75), (53, 60, 63, 69, 74, 75), (53, 60, 63, 69, 74, 75), (53, 60, 63, 69, 74, 75), (53, 60, 63, 69, 73, 74, 75), (53, 60, 63, 69, 73, 74, 75), (56, 62, 67, 74), (56, 62, 67, 74), (56, 62, 67, 74), (56, 62, 67, 74), (46, 56, 62, 67, 74), (46,), (46,), (46,), (46,), (46,), (46, 77, 89), (46, 77, 89), (46, 77, 89), (54, 60, 65, 77, 89), (54, 60, 65, 77, 89), (54, 60, 65, 77, 89), (54, 60, 65, 77, 89), (54, 60, 65, 74, 77, 86, 89), (54, 60, 65, 74, 77, 86, 89), (54, 60, 65, 74, 77, 86, 89), (54, 60, 65, 74, 77, 86, 89), (54, 60, 65, 74, 77, 86, 89), (54, 60, 65, 74, 77, 86, 89), (44, 54, 60, 65, 74, 77, 86, 89), (44, 54, 60, 65, 74, 77, 86, 89), (59, 62), (43, 53, 59, 62, 67), (43, 53, 59, 67), (43, 53, 59, 67), (43, 53, 59, 67), (43, 53, 56, 59, 62, 65), (43, 53, 56, 59, 62, 65), (43, 53, 56, 59, 62, 65), (43, 53, 56, 59, 62, 65), (43, 53, 56, 59, 62, 65), (43, 53, 56, 59, 62, 65), (43, 50, 53, 56, 59, 62, 65), (43, 50, 53, 56, 59, 62, 65), (43, 47, 50, 53, 56, 59, 62, 65), (47, 56, 59, 62), (47, 56, 59, 62), (47, 56, 59, 62), (47, 56, 59, 62), (47, 56, 59, 62), (47, 56, 59, 62), (47, 56, 59, 62, 65), (47, 56, 59, 62, 65), (47, 56, 59, 62, 65), (47, 56, 59, 62, 65), (47, 56, 59, 62, 65), (47, 56, 59, 62, 65), (48, 55, 60, 63), (48, 63), (48, 63), (48, 63), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 63, 74), (48, 62, 63, 67, 74), (48, 62, 63, 67, 74), (48, 62, 63, 67, 74), (48, 62, 63, 67, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (48, 60, 62, 63, 67, 72, 74), (46, 63, 67), (46, 55, 63, 67), (46, 55, 63, 67), (46, 55, 63, 67), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (46, 55, 63, 67, 74, 84, 86), (45, 55, 63, 67, 74), (55,), (55,), (55,), (55, 62, 74, 86), (55, 62, 74, 86), (55, 62, 74, 86), (55, 62, 74, 86), (55, 62, 63, 74, 75, 86, 87), (55, 62, 63, 74, 75, 86, 87), (55, 62, 63, 74, 75, 86, 87), (86,), (62, 74, 86), (62, 74, 86), (62, 74, 86), (62, 63, 74, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (62, 63, 74, 75, 86, 87), (50, 60, 75, 87), (50, 60, 75, 87), (50, 60, 75, 87), (50, 60, 75, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (50, 54, 60, 63, 69, 74, 75, 86, 87), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (43, 50, 57, 58, 62, 69), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60), (45, 51, 55, 60), (45, 51, 55, 60), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (45, 51, 55, 60, 63), (50, 53, 58, 62), (43, 50, 53, 58, 62), (43, 50), (43, 50), (43, 50, 55), (43, 50, 55), (43, 50, 55), (43, 50, 55), (43, 50, 55), (43, 50, 55, 74), (43, 50, 55, 74), (43, 50, 55, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 74), (43, 50, 55, 58, 62, 67, 70, 72, 74), (43, 50, 55, 58, 62, 67, 70, 72, 74), (43, 50, 55, 58, 62, 67, 70, 72, 74, 75), (43, 50, 55, 58, 62, 67, 70, 72, 74, 75), (60, 63, 68, 72, 75, 79), (60, 63, 68, 72, 75, 79), (60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 72, 75, 79), (58, 60, 63, 68, 70, 72, 75, 79), (58, 60, 63, 68, 70, 72, 75, 79), (58, 60, 63, 68, 70, 72, 75, 79), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (46, 53, 56, 59, 62), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62), (39, 46, 55, 58, 62)]\n",
      "(128, 1602)\n"
     ]
    }
   ],
   "source": [
    "# ori_full = all_song_tokenised[song_idx][seq_start_at:].tolist()\n",
    "ori_full = inf_song[:sequence_length+1000]\n",
    "ori_full = convertToRoll(ori_full)\n",
    "ori_full = piano_roll_to_pretty_midi(ori_full.T, fs=fs)\n",
    "midi_ori_full_path = output_path+f\"orifull-gpt-v3-{name}.mid\"\n",
    "if midi_ori_full_path is not None:\n",
    "        ori_full.write(midi_ori_full_path)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "7iw9iWCXjLhJ",
    "ZZOK3emyh0D0",
    "gh3RhUvE8cKt",
    "vNXWdP8DgnUK"
   ],
   "name": "TestEnvForMusicGen.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}