dataproc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c17f1525-bfc4-4da5-a9ec-78a8779a4c82",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/icarrera/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/icarrera/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "# Load abstracts DataFrame\n",
    "df = pd.read_csv(filepath_or_buffer=\"data/abstracts.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "362aea5b-810d-4ef7-8ff0-d0faa1108fb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.stem import PorterStemmer\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "# Text cleaning function\n",
    "def clean_text(text):\n",
    "    # Remove special characters and numbers\n",
    "    text = re.sub('[^a-zA-Z]', ' ', text)\n",
    "    # Convert to lowercase\n",
    "    text = text.lower()    \n",
    "    # Remove leading and trailing white spaces\n",
    "    text = text.strip()\n",
    "    # Replace multiple spaces with a single space\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    # Tokenize text\n",
    "    words = word_tokenize(text)\n",
    "    # Eliminate stopwords y stem words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    stemmed_words = [stemmer.stem(word) for word in words if word not in stop_words]\n",
    "    # Join words into sentences\n",
    "    cleaned_text = ' '.join(stemmed_words)\n",
    "    return cleaned_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c2c772c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess abstracts\n",
    "df.insert(1, \"abstract_pr\", df['abstract'].apply(clean_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3d2685ae-5bcb-4a46-8512-abc872246b09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>abstract_pr</th>\n",
       "      <th>cell_line</th>\n",
       "      <th>pubmedid</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tripl neg breast cancer tnbc highli aggress su...</td>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>35715860</td>\n",
       "      <td>Tumor-associated macrophages promote epithelia...</td>\n",
       "      <td>Triple-negative breast cancer (TNBC) is a high...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>thp repres leukemia cell line regist four diff...</td>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>35373342</td>\n",
       "      <td>THP-1 reference data: Proposal of an in vitro ...</td>\n",
       "      <td>THP-1 is a representative leukemia cell line a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>pattern hydroxi n trichlorophenyl naphthamid t...</td>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>35745634</td>\n",
       "      <td>Antistaphylococcal Activities and ADME-Related...</td>\n",
       "      <td>Pattern 1-hydroxy-N-(2,4,5-trichlorophenyl)-2-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>seri eleven benzyl intermedi eleven target com...</td>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>36232947</td>\n",
       "      <td>Study of Biological Activities and ADMET-Relat...</td>\n",
       "      <td>A series of eleven benzylated intermediates an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>seri thirti two anilid trifluoromethyl cinnam ...</td>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>36499415</td>\n",
       "      <td>Trifluoromethylcinnamanilide Michael Acceptors...</td>\n",
       "      <td>A series of thirty-two anilides of 3-(trifluor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266785</th>\n",
       "      <td>8</td>\n",
       "      <td>crispr cluster regularli interspac short palin...</td>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>33969526</td>\n",
       "      <td>In vivo CRISPR screening for novel noncoding R...</td>\n",
       "      <td>CRISPR (clustered regularly interspaced short ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266786</th>\n",
       "      <td>9</td>\n",
       "      <td>traumat injuri often result axon sever initi o...</td>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>29577375</td>\n",
       "      <td>Ca2+/calmodulin-dependent protein kinase II an...</td>\n",
       "      <td>Traumatic injury often results in axonal sever...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266787</th>\n",
       "      <td>10</td>\n",
       "      <td>oxid stress implic varieti neurodegen disord a...</td>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>32954502</td>\n",
       "      <td>Noradrenaline protects neurons against H&lt;sub&gt;2...</td>\n",
       "      <td>Oxidative stress has been implicated in a vari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266788</th>\n",
       "      <td>11</td>\n",
       "      <td>primari blast injuri caus direct impact overpr...</td>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>36200530</td>\n",
       "      <td>Development of a novel bioengineered 3D brain-...</td>\n",
       "      <td>Primary blast injury is caused by the direct i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266789</th>\n",
       "      <td>12</td>\n",
       "      <td>tomosyn protein encod syntaxin bind protein st...</td>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>32133675</td>\n",
       "      <td>Tomosyn regulates the small RhoA GTPase to con...</td>\n",
       "      <td>Tomosyn, a protein encoded by syntaxin-1-bindi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>266790 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        index                                        abstract_pr  cell_line  \\\n",
       "0           0  tripl neg breast cancer tnbc highli aggress su...  CVCL_0006   \n",
       "1           1  thp repres leukemia cell line regist four diff...  CVCL_0006   \n",
       "2           2  pattern hydroxi n trichlorophenyl naphthamid t...  CVCL_0006   \n",
       "3           3  seri eleven benzyl intermedi eleven target com...  CVCL_0006   \n",
       "4           4  seri thirti two anilid trifluoromethyl cinnam ...  CVCL_0006   \n",
       "...       ...                                                ...        ...   \n",
       "266785      8  crispr cluster regularli interspac short palin...  CVCL_ZZ97   \n",
       "266786      9  traumat injuri often result axon sever initi o...  CVCL_ZZ97   \n",
       "266787     10  oxid stress implic varieti neurodegen disord a...  CVCL_ZZ97   \n",
       "266788     11  primari blast injuri caus direct impact overpr...  CVCL_ZZ97   \n",
       "266789     12  tomosyn protein encod syntaxin bind protein st...  CVCL_ZZ97   \n",
       "\n",
       "        pubmedid                                              title  \\\n",
       "0       35715860  Tumor-associated macrophages promote epithelia...   \n",
       "1       35373342  THP-1 reference data: Proposal of an in vitro ...   \n",
       "2       35745634  Antistaphylococcal Activities and ADME-Related...   \n",
       "3       36232947  Study of Biological Activities and ADMET-Relat...   \n",
       "4       36499415  Trifluoromethylcinnamanilide Michael Acceptors...   \n",
       "...          ...                                                ...   \n",
       "266785  33969526  In vivo CRISPR screening for novel noncoding R...   \n",
       "266786  29577375  Ca2+/calmodulin-dependent protein kinase II an...   \n",
       "266787  32954502  Noradrenaline protects neurons against H<sub>2...   \n",
       "266788  36200530  Development of a novel bioengineered 3D brain-...   \n",
       "266789  32133675  Tomosyn regulates the small RhoA GTPase to con...   \n",
       "\n",
       "                                                 abstract  \n",
       "0       Triple-negative breast cancer (TNBC) is a high...  \n",
       "1       THP-1 is a representative leukemia cell line a...  \n",
       "2       Pattern 1-hydroxy-N-(2,4,5-trichlorophenyl)-2-...  \n",
       "3       A series of eleven benzylated intermediates an...  \n",
       "4       A series of thirty-two anilides of 3-(trifluor...  \n",
       "...                                                   ...  \n",
       "266785  CRISPR (clustered regularly interspaced short ...  \n",
       "266786  Traumatic injury often results in axonal sever...  \n",
       "266787  Oxidative stress has been implicated in a vari...  \n",
       "266788  Primary blast injury is caused by the direct i...  \n",
       "266789  Tomosyn, a protein encoded by syntaxin-1-bindi...  \n",
       "\n",
       "[266790 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d94a4de-a277-4a80-afaa-b8319ad63a12",
   "metadata": {},
   "source": [
    "## Tf-idf 1 (Term frequency – Inverse document frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6a7e30a8-1c28-4231-b87e-d6f549ac2e4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "# Apply TF-IDF to \"abstract_pr\"\n",
    "tfidf_matrix = vectorizer.fit_transform(df['abstract_pr'])\n",
    "# Extract features\n",
    "features = vectorizer.get_feature_names_out()\n",
    "# Convert the TF-IDF matrix to a DataFrame\n",
    "tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd554577-204d-4fa8-8b6a-42791147a8aa",
   "metadata": {},
   "source": [
    "## Tf-idf 2 (Term frequency – Inverse document frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eb8c0f91-c47e-4d20-96f4-76c1d91aeb23",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# Create an instance of TfidfVectorizer /Crear una instancia de TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer()\n",
    "# Fit the vectorizer on the 'abstract_p' column/# Ajustar el vectorizador en la columna 'abstract_p\n",
    "vectorizer.fit(df['abstract_pr'])\n",
    "# Transform the 'abstract_p' column into TF-IDF features / Transformar la columna 'abstract_p' en características TF-IDF\n",
    "tfidf_matrix = vectorizer.transform(df['abstract_pr'])\n",
    "# Convert the TF-IDF matrix to a DataFrame / Convertir la matriz TF-IDF en un DataFrame\n",
    "tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=vectorizer.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a3a287b7-6bb3-4781-bf98-82043593bdf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aa</th>\n",
       "      <th>aaa</th>\n",
       "      <th>aaaf</th>\n",
       "      <th>aaatcaatatt</th>\n",
       "      <th>aab</th>\n",
       "      <th>aabcl</th>\n",
       "      <th>aac</th>\n",
       "      <th>aacocf</th>\n",
       "      <th>aacr</th>\n",
       "      <th>aacrjourn</th>\n",
       "      <th>...</th>\n",
       "      <th>zymosan</th>\n",
       "      <th>zynaxi</th>\n",
       "      <th>zyx</th>\n",
       "      <th>zyxin</th>\n",
       "      <th>zz</th>\n",
       "      <th>zzr</th>\n",
       "      <th>zzui</th>\n",
       "      <th>zzuneui</th>\n",
       "      <th>zzusahi</th>\n",
       "      <th>zzw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266785</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266786</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266787</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266788</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266789</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>266790 rows × 57470 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         aa  aaa  aaaf  aaatcaatatt  aab  aabcl  aac  aacocf  aacr  aacrjourn  \\\n",
       "0       0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "1       0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "2       0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "3       0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "4       0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "...     ...  ...   ...          ...  ...    ...  ...     ...   ...        ...   \n",
       "266785  0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "266786  0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "266787  0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "266788  0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "266789  0.0  0.0   0.0          0.0  0.0    0.0  0.0     0.0   0.0        0.0   \n",
       "\n",
       "        ...  zymosan  zynaxi  zyx  zyxin   zz  zzr  zzui  zzuneui  zzusahi  \\\n",
       "0       ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "1       ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "2       ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "3       ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "4       ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "...     ...      ...     ...  ...    ...  ...  ...   ...      ...      ...   \n",
       "266785  ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "266786  ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "266787  ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "266788  ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "266789  ...      0.0     0.0  0.0    0.0  0.0  0.0   0.0      0.0      0.0   \n",
       "\n",
       "        zzw  \n",
       "0       0.0  \n",
       "1       0.0  \n",
       "2       0.0  \n",
       "3       0.0  \n",
       "4       0.0  \n",
       "...     ...  \n",
       "266785  0.0  \n",
       "266786  0.0  \n",
       "266787  0.0  \n",
       "266788  0.0  \n",
       "266789  0.0  \n",
       "\n",
       "[266790 rows x 57470 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "538c4d02-4559-48af-8a4e-e12fbf475248",
   "metadata": {},
   "source": [
    "## REDUCCIÓN DE DIMENSIONES CON PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e5ba9948-75ff-4c23-b01d-4d38bce9dd8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               0         1         2         3         4\n",
      "0       0.057996 -0.001682 -0.017167  0.018858  0.038214\n",
      "1       0.019032  0.013439  0.004107  0.012282 -0.014082\n",
      "2       0.005033  0.033706 -0.002056  0.030673 -0.024185\n",
      "3       0.006142  0.043782 -0.021622  0.018907 -0.012791\n",
      "4      -0.006364  0.057346 -0.015441  0.029560 -0.012861\n",
      "...          ...       ...       ...       ...       ...\n",
      "266785  0.120181  0.280024 -0.166574  0.146169 -0.292266\n",
      "266786 -0.501375 -0.412417 -0.129923 -0.078504 -0.073478\n",
      "266787 -0.162225  0.164514  0.423831  0.029649  0.103834\n",
      "266788 -0.112572  0.187052  0.314679 -0.272013  0.093490\n",
      "266789 -0.059619  0.149818 -0.087470  0.015213  0.187121\n",
      "\n",
      "[266790 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    " # Step 2: Reduce dimensions using PCA\n",
    "pca = PCA(n_components=5)  # Specify the number of desired dimensions\n",
    "tfidf_pca = pca.fit_transform(tfidf_matrix.toarray())\n",
    "\n",
    "# Create a new DataFrame with the PCA results\n",
    "df_pca = pd.DataFrame(data=tfidf_pca)\n",
    "\n",
    "# Print the final DataFrame with TF-IDF and PCA results\n",
    "print(df_pca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "abcb2112-fa44-43b2-894c-4246aa420f19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cell_line</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>0.057996</td>\n",
       "      <td>-0.001682</td>\n",
       "      <td>-0.017167</td>\n",
       "      <td>0.018858</td>\n",
       "      <td>0.038214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>0.019032</td>\n",
       "      <td>0.013439</td>\n",
       "      <td>0.004107</td>\n",
       "      <td>0.012282</td>\n",
       "      <td>-0.014082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>0.005033</td>\n",
       "      <td>0.033706</td>\n",
       "      <td>-0.002056</td>\n",
       "      <td>0.030673</td>\n",
       "      <td>-0.024185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>0.006142</td>\n",
       "      <td>0.043782</td>\n",
       "      <td>-0.021622</td>\n",
       "      <td>0.018907</td>\n",
       "      <td>-0.012791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>-0.006364</td>\n",
       "      <td>0.057346</td>\n",
       "      <td>-0.015441</td>\n",
       "      <td>0.029560</td>\n",
       "      <td>-0.012861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266785</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>0.120181</td>\n",
       "      <td>0.280024</td>\n",
       "      <td>-0.166574</td>\n",
       "      <td>0.146169</td>\n",
       "      <td>-0.292266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266786</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>-0.501375</td>\n",
       "      <td>-0.412417</td>\n",
       "      <td>-0.129923</td>\n",
       "      <td>-0.078504</td>\n",
       "      <td>-0.073478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266787</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>-0.162225</td>\n",
       "      <td>0.164514</td>\n",
       "      <td>0.423831</td>\n",
       "      <td>0.029649</td>\n",
       "      <td>0.103834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266788</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>-0.112572</td>\n",
       "      <td>0.187052</td>\n",
       "      <td>0.314679</td>\n",
       "      <td>-0.272013</td>\n",
       "      <td>0.093490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266789</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>-0.059619</td>\n",
       "      <td>0.149818</td>\n",
       "      <td>-0.087470</td>\n",
       "      <td>0.015213</td>\n",
       "      <td>0.187121</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>266790 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        cell_line         0         1         2         3         4\n",
       "0       CVCL_0006  0.057996 -0.001682 -0.017167  0.018858  0.038214\n",
       "1       CVCL_0006  0.019032  0.013439  0.004107  0.012282 -0.014082\n",
       "2       CVCL_0006  0.005033  0.033706 -0.002056  0.030673 -0.024185\n",
       "3       CVCL_0006  0.006142  0.043782 -0.021622  0.018907 -0.012791\n",
       "4       CVCL_0006 -0.006364  0.057346 -0.015441  0.029560 -0.012861\n",
       "...           ...       ...       ...       ...       ...       ...\n",
       "266785  CVCL_ZZ97  0.120181  0.280024 -0.166574  0.146169 -0.292266\n",
       "266786  CVCL_ZZ97 -0.501375 -0.412417 -0.129923 -0.078504 -0.073478\n",
       "266787  CVCL_ZZ97 -0.162225  0.164514  0.423831  0.029649  0.103834\n",
       "266788  CVCL_ZZ97 -0.112572  0.187052  0.314679 -0.272013  0.093490\n",
       "266789  CVCL_ZZ97 -0.059619  0.149818 -0.087470  0.015213  0.187121\n",
       "\n",
       "[266790 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate the PCA results with the original DataFrame\n",
    "df_final = pd.concat([df['cell_line'], df_pca], axis=1)\n",
    "df_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44627c6e-7520-4b11-890f-078a8d430249",
   "metadata": {},
   "source": [
    "## SVDD APLICANDO PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f182a635-fd7f-4e39-b538-56b307ff4648",
   "metadata": {},
   "outputs": [],
   "source": [
    "# aplicando pca\n",
    "from sklearn.svm import OneClassSVM\n",
    "import numpy as np\n",
    "\n",
    "# Step 3: Apply SVDD for outlier detection and subject identification\n",
    "\n",
    "subjects = df['cell_line'].unique()\n",
    "\n",
    "subject_centers_pca = {}\n",
    "subject_radii_pca = {}\n",
    "\n",
    "for subject in subjects:\n",
    "    subject_indices = df[df['cell_line'] == subject].index\n",
    "\n",
    "    # Extract TF-IDF vectors for the subject's texts\n",
    "    subject_tfidf_matrix_pca = tfidf_pca[subject_indices]\n",
    "\n",
    "    # Apply SVDD (OneClassSVM) for the subject's texts\n",
    "    model = OneClassSVM(kernel='rbf', gamma='scale')\n",
    "    model.fit(subject_tfidf_matrix_pca)\n",
    "\n",
    "    # Filter inliers for the subject\n",
    "    inlier_indices = subject_indices[model.predict(subject_tfidf_matrix_pca) == 1]\n",
    "    inliers_df = df.loc[inlier_indices]\n",
    "\n",
    "    # Calculate the center of the vectors for the subject\n",
    "    subject_center_pca = np.mean(subject_tfidf_matrix_pca, axis=0)\n",
    "    subject_centers_pca[subject] = subject_center_pca\n",
    "\n",
    "    # Calculate the radius of the sphere that describes the subject\n",
    "    subject_radius_pca = np.max(np.linalg.norm(subject_tfidf_matrix_pca - subject_center_pca, axis=1))\n",
    "    subject_radii_pca[subject] = subject_radius_pca\n",
    "\n",
    "# Filter the DataFrame to include only the inliers\n",
    "df_inliers = df.loc[inliers_df.index]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "525fa95b-27fb-4563-ad61-bc55fa244a9e",
   "metadata": {},
   "source": [
    "## CENTROIDES CON PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "748bbdf0-1536-44b6-8db0-0fb814864aad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cell_line</th>\n",
       "      <th>center</th>\n",
       "      <th>radius</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CVCL_0006</td>\n",
       "      <td>[0.008978779111048439, 0.023425353761767027, -...</td>\n",
       "      <td>0.070312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CVCL_0024</td>\n",
       "      <td>[0.02337680012648178, -0.011364185264599026, -...</td>\n",
       "      <td>0.052139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CVCL_0043</td>\n",
       "      <td>[0.011487527481506488, 0.025402494463191438, -...</td>\n",
       "      <td>0.043485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CVCL_0046</td>\n",
       "      <td>[0.015073659155474158, 0.014286755609635382, 0...</td>\n",
       "      <td>0.053614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CVCL_0067</td>\n",
       "      <td>[0.014743355613286553, 0.010479769992344644, 0...</td>\n",
       "      <td>0.074357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21839</th>\n",
       "      <td>CVCL_ZZ76</td>\n",
       "      <td>[0.013231947428114926, -0.05494187680642152, -...</td>\n",
       "      <td>0.894384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21840</th>\n",
       "      <td>CVCL_ZZ79</td>\n",
       "      <td>[-0.0006030672469505027, -0.010575210762174872...</td>\n",
       "      <td>0.889204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21841</th>\n",
       "      <td>CVCL_ZZ83</td>\n",
       "      <td>[-0.006925923016179529, -0.0005921389329829629...</td>\n",
       "      <td>0.892903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21842</th>\n",
       "      <td>CVCL_ZZ92</td>\n",
       "      <td>[0.00901969607112853, -0.023780498723534913, -...</td>\n",
       "      <td>0.872042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21843</th>\n",
       "      <td>CVCL_ZZ97</td>\n",
       "      <td>[-0.006925923016179448, -0.0005921389329829501...</td>\n",
       "      <td>0.892903</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21844 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       cell_line                                             center    radius\n",
       "0      CVCL_0006  [0.008978779111048439, 0.023425353761767027, -...  0.070312\n",
       "1      CVCL_0024  [0.02337680012648178, -0.011364185264599026, -...  0.052139\n",
       "2      CVCL_0043  [0.011487527481506488, 0.025402494463191438, -...  0.043485\n",
       "3      CVCL_0046  [0.015073659155474158, 0.014286755609635382, 0...  0.053614\n",
       "4      CVCL_0067  [0.014743355613286553, 0.010479769992344644, 0...  0.074357\n",
       "...          ...                                                ...       ...\n",
       "21839  CVCL_ZZ76  [0.013231947428114926, -0.05494187680642152, -...  0.894384\n",
       "21840  CVCL_ZZ79  [-0.0006030672469505027, -0.010575210762174872...  0.889204\n",
       "21841  CVCL_ZZ83  [-0.006925923016179529, -0.0005921389329829629...  0.892903\n",
       "21842  CVCL_ZZ92  [0.00901969607112853, -0.023780498723534913, -...  0.872042\n",
       "21843  CVCL_ZZ97  [-0.006925923016179448, -0.0005921389329829501...  0.892903\n",
       "\n",
       "[21844 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Crear una lista para almacenar los resultados\n",
    "result_data = []\n",
    "\n",
    "# Llenar la lista con los resultados\n",
    "for subject in subjects:\n",
    "    center = np.squeeze(subject_centers_pca[subject])\n",
    "    radius = np.squeeze(subject_radii_pca[subject])\n",
    "    result_data.append({'cell_line': subject, 'center': center, 'radius': radius})\n",
    "\n",
    "# Crear el dataframe a partir de la lista de resultados\n",
    "result_df_pca = pd.DataFrame(result_data)\n",
    "\n",
    "# Imprimir el dataframe de resultados\n",
    "result_df_pca\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d05783f3-96a8-421d-b4f0-ce19f525a6a5",
   "metadata": {},
   "source": [
    "## MATRIZ DISTANCIA CON PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ced5e378-0c6a-4013-9c12-4e84536e0aa3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00000000e+00 4.22834569e-02 9.39595392e-03 ... 3.59178986e-02\n",
      "  5.18959098e-02 3.59178986e-02]\n",
      " [4.22834569e-02 0.00000000e+00 4.17688219e-02 ... 3.29348369e-02\n",
      "  3.68771553e-02 3.29348369e-02]\n",
      " [9.39595392e-03 4.17688219e-02 0.00000000e+00 ... 3.58269400e-02\n",
      "  5.28737856e-02 3.58269400e-02]\n",
      " ...\n",
      " [3.59178986e-02 3.29348369e-02 3.58269400e-02 ... 0.00000000e+00\n",
      "  4.38874887e-02 9.38804520e-17]\n",
      " [5.18959098e-02 3.68771553e-02 5.28737856e-02 ... 4.38874887e-02\n",
      "  0.00000000e+00 4.38874887e-02]\n",
      " [3.59178986e-02 3.29348369e-02 3.58269400e-02 ... 9.38804520e-17\n",
      "  4.38874887e-02 0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy.spatial.distance import euclidean\n",
    "\n",
    "n = len(result_data)  # Number of observations\n",
    "distance_matrix = np.zeros((n, n))  # Initialize an empty distance matrix\n",
    "\n",
    "# Calculate pairwise distances using a for loop\n",
    "for i in range(n):\n",
    "    for j in range(i + 1, n):\n",
    "        center_i = np.asarray(result_data[i]['center'])\n",
    "        center_j = np.asarray(result_data[j]['center'])\n",
    "        distance = euclidean(center_i.flatten(), center_j.flatten())                  # Euclidean distance between flattened 'center' values\n",
    "        distance_matrix[i, j] = distance\n",
    "        distance_matrix[j, i] = distance\n",
    "\n",
    "# Print the distance matrix\n",
    "print(distance_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "10a392bc-4c04-4848-8847-84ac3e5180af",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_5474/763666440.py:6: ClusterWarning: scipy.cluster: The symmetric non-negative hollow observation matrix looks suspiciously like an uncondensed distance matrix\n",
      "  Z = linkage(distance_matrix, method='average')\n"
     ]
    },
    {
     "ename": "RecursionError",
     "evalue": "maximum recursion depth exceeded while getting the str of an object",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRecursionError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[12], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;66;03m# Plot the dendrogram\u001b[39;00m\n\u001b[1;32m      9\u001b[0m plt\u001b[38;5;241m.\u001b[39mfigure(figsize\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m16\u001b[39m, \u001b[38;5;241m9\u001b[39m))\n\u001b[0;32m---> 10\u001b[0m \u001b[43mdendrogram\u001b[49m\u001b[43m(\u001b[49m\u001b[43mZ\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     11\u001b[0m plt\u001b[38;5;241m.\u001b[39mxlabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSubjects\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     12\u001b[0m plt\u001b[38;5;241m.\u001b[39mylabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDistance\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3353\u001b[0m, in \u001b[0;36mdendrogram\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, no_plot, no_labels, leaf_font_size, leaf_rotation, leaf_label_func, show_contracted, link_color_func, ax, above_threshold_color)\u001b[0m\n\u001b[1;32m   3350\u001b[0m \u001b[38;5;66;03m# Empty list will be filled in _dendrogram_calculate_info\u001b[39;00m\n\u001b[1;32m   3351\u001b[0m contraction_marks \u001b[38;5;241m=\u001b[39m [] \u001b[38;5;28;01mif\u001b[39;00m show_contracted \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 3353\u001b[0m \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3354\u001b[0m \u001b[43m    \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3355\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3356\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3357\u001b[0m \u001b[43m    \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3358\u001b[0m \u001b[43m    \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3359\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3360\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3361\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3362\u001b[0m \u001b[43m    \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3363\u001b[0m \u001b[43m    \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3364\u001b[0m \u001b[43m    \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.0\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3365\u001b[0m \u001b[43m    \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3366\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3367\u001b[0m \u001b[43m    \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3368\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3369\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3370\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3371\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3372\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3373\u001b[0m \u001b[43m    \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3374\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3375\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3376\u001b[0m \u001b[43m    \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3378\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_plot:\n\u001b[1;32m   3379\u001b[0m     mh \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(Z[:, \u001b[38;5;241m2\u001b[39m])\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3658\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3654\u001b[0m     currently_below_threshold[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m   3655\u001b[0m     c \u001b[38;5;241m=\u001b[39m _link_line_colors[current_color[\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m   3657\u001b[0m (uivb, uwb, ubh, ubmd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3658\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3659\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3660\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3661\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3662\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3663\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3664\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3665\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3666\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3667\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3668\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mub\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43muwa\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3669\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3673\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3675\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3676\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3677\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3679\u001b[0m max_dist \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(uamd, ubmd, h)\n\u001b[1;32m   3681\u001b[0m icoord_list\u001b[38;5;241m.\u001b[39mappend([uiva, uiva, uivb, uivb])\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3658\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3654\u001b[0m     currently_below_threshold[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m   3655\u001b[0m     c \u001b[38;5;241m=\u001b[39m _link_line_colors[current_color[\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m   3657\u001b[0m (uivb, uwb, ubh, ubmd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3658\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3659\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3660\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3661\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3662\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3663\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3664\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3665\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3666\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3667\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3668\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mub\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43muwa\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3669\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3673\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3675\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3676\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3677\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3679\u001b[0m max_dist \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(uamd, ubmd, h)\n\u001b[1;32m   3681\u001b[0m icoord_list\u001b[38;5;241m.\u001b[39mappend([uiva, uiva, uivb, uivb])\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3625\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3621\u001b[0m     ub \u001b[38;5;241m=\u001b[39m ab\n\u001b[1;32m   3623\u001b[0m \u001b[38;5;66;03m# Updated iv variable and the amount of space used.\u001b[39;00m\n\u001b[1;32m   3624\u001b[0m (uiva, uwa, uah, uamd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3625\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3626\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3627\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3628\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3629\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3630\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3631\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3632\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3633\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3634\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3635\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mua\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3636\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3637\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3638\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3639\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3640\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3641\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3642\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3643\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3644\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3646\u001b[0m h \u001b[38;5;241m=\u001b[39m Z[i \u001b[38;5;241m-\u001b[39m n, \u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m   3647\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m h \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m color_threshold \u001b[38;5;129;01mor\u001b[39;00m color_threshold \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3625\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3621\u001b[0m     ub \u001b[38;5;241m=\u001b[39m ab\n\u001b[1;32m   3623\u001b[0m \u001b[38;5;66;03m# Updated iv variable and the amount of space used.\u001b[39;00m\n\u001b[1;32m   3624\u001b[0m (uiva, uwa, uah, uamd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3625\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3626\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3627\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3628\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3629\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3630\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3631\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3632\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3633\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3634\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3635\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mua\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3636\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3637\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3638\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3639\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3640\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3641\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3642\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3643\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3644\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3646\u001b[0m h \u001b[38;5;241m=\u001b[39m Z[i \u001b[38;5;241m-\u001b[39m n, \u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m   3647\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m h \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m color_threshold \u001b[38;5;129;01mor\u001b[39;00m color_threshold \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "    \u001b[0;31m[... skipping similar frames: _dendrogram_calculate_info at line 3658 (2961 times), _dendrogram_calculate_info at line 3625 (2 times)]\u001b[0m\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3658\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3654\u001b[0m     currently_below_threshold[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m   3655\u001b[0m     c \u001b[38;5;241m=\u001b[39m _link_line_colors[current_color[\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m   3657\u001b[0m (uivb, uwb, ubh, ubmd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3658\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3659\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3660\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3661\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3662\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3663\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3664\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3665\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3666\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3667\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3668\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mub\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43muwa\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3669\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3673\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3675\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3676\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3677\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3679\u001b[0m max_dist \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(uamd, ubmd, h)\n\u001b[1;32m   3681\u001b[0m icoord_list\u001b[38;5;241m.\u001b[39mappend([uiva, uiva, uivb, uivb])\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3625\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3621\u001b[0m     ub \u001b[38;5;241m=\u001b[39m ab\n\u001b[1;32m   3623\u001b[0m \u001b[38;5;66;03m# Updated iv variable and the amount of space used.\u001b[39;00m\n\u001b[1;32m   3624\u001b[0m (uiva, uwa, uah, uamd) \u001b[38;5;241m=\u001b[39m \\\n\u001b[0;32m-> 3625\u001b[0m     \u001b[43m_dendrogram_calculate_info\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3626\u001b[0m \u001b[43m        \u001b[49m\u001b[43mZ\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3627\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3628\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3629\u001b[0m \u001b[43m        \u001b[49m\u001b[43mget_leaves\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_leaves\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3630\u001b[0m \u001b[43m        \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3631\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3632\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcount_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3633\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdistance_sort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdistance_sort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3634\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_leaf_counts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3635\u001b[0m \u001b[43m        \u001b[49m\u001b[43mi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mua\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3636\u001b[0m \u001b[43m        \u001b[49m\u001b[43micoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43micoord_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3637\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdcoord_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdcoord_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3638\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrent_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrent_color\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3639\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_list\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolor_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3640\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurrently_below_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3641\u001b[0m \u001b[43m        \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3642\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontraction_marks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontraction_marks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3643\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlink_color_func\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlink_color_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3644\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabove_threshold_color\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mabove_threshold_color\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3646\u001b[0m h \u001b[38;5;241m=\u001b[39m Z[i \u001b[38;5;241m-\u001b[39m n, \u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m   3647\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m h \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m color_threshold \u001b[38;5;129;01mor\u001b[39;00m color_threshold \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3555\u001b[0m, in \u001b[0;36m_dendrogram_calculate_info\u001b[0;34m(Z, p, truncate_mode, color_threshold, get_leaves, orientation, labels, count_sort, distance_sort, show_leaf_counts, i, iv, ivl, n, icoord_list, dcoord_list, lvs, mhr, current_color, color_list, currently_below_threshold, leaf_label_func, level, contraction_marks, link_color_func, above_threshold_color)\u001b[0m\n\u001b[1;32m   3551\u001b[0m \u001b[38;5;66;03m# Otherwise, only truncate if we have a leaf node.\u001b[39;00m\n\u001b[1;32m   3552\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m   3553\u001b[0m \u001b[38;5;66;03m# Only place leaves if they correspond to original observations.\u001b[39;00m\n\u001b[1;32m   3554\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m i \u001b[38;5;241m<\u001b[39m n:\n\u001b[0;32m-> 3555\u001b[0m     \u001b[43m_append_singleton_leaf_node\u001b[49m\u001b[43m(\u001b[49m\u001b[43mZ\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlvs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mivl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3556\u001b[0m \u001b[43m                                \u001b[49m\u001b[43mleaf_label_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m (iv \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m5.0\u001b[39m, \u001b[38;5;241m10.0\u001b[39m, \u001b[38;5;241m0.0\u001b[39m, \u001b[38;5;241m0.0\u001b[39m)\n\u001b[1;32m   3559\u001b[0m \u001b[38;5;66;03m# !!! Otherwise, we don't have a leaf node, so work on plotting a\u001b[39;00m\n\u001b[1;32m   3560\u001b[0m \u001b[38;5;66;03m# non-leaf node.\u001b[39;00m\n\u001b[1;32m   3561\u001b[0m \u001b[38;5;66;03m# Actual indices of a and b\u001b[39;00m\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/scipy/cluster/hierarchy.py:3433\u001b[0m, in \u001b[0;36m_append_singleton_leaf_node\u001b[0;34m(Z, p, n, level, lvs, ivl, leaf_label_func, i, labels)\u001b[0m\n\u001b[1;32m   3430\u001b[0m     ivl\u001b[38;5;241m.\u001b[39mappend(labels[\u001b[38;5;28mint\u001b[39m(i \u001b[38;5;241m-\u001b[39m n)])\n\u001b[1;32m   3431\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   3432\u001b[0m     \u001b[38;5;66;03m# Otherwise, use the id as the label for the leaf.x\u001b[39;00m\n\u001b[0;32m-> 3433\u001b[0m     ivl\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mi\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n",
      "\u001b[0;31mRecursionError\u001b[0m: maximum recursion depth exceeded while getting the str of an object"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x900 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.cluster.hierarchy import linkage, dendrogram\n",
    "\n",
    "# Compute the linkage matrix\n",
    "Z = linkage(distance_matrix, method='average')\n",
    "\n",
    "# Plot the dendrogram\n",
    "plt.figure(figsize=(16, 9))\n",
    "dendrogram(Z)\n",
    "plt.xlabel('Subjects')\n",
    "plt.ylabel('Distance')\n",
    "plt.title('Dendrogram')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c7b3ff7-56f1-4a67-96cc-223271dcda7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_5474/251422438.py:6: ClusterWarning: scipy.cluster: The symmetric non-negative hollow observation matrix looks suspiciously like an uncondensed distance matrix\n",
      "  Z = linkage(distance_matrix, method='average')\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.cluster.hierarchy import linkage, dendrogram\n",
    "\n",
    "# Compute the linkage matrix\n",
    "Z = linkage(distance_matrix, method='average')\n",
    "\n",
    "# Plot the dendrogram\n",
    "plt.figure(figsize=(1, 6))\n",
    "dendrogram(Z)\n",
    "plt.xlabel('Subjects')\n",
    "plt.ylabel('Distance')\n",
    "plt.title('Dendrogram')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1bfc42c-e88d-47e0-8ab5-0a37e53a0386",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}