tutorials/tutorial_hierarchical_topic_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://raw.githubusercontent.com/BobXWu/TopMost/master/data/NYT.zip\n",
      "Using downloaded and verified file: ./datasets/NYT.zip\n"
     ]
    }
   ],
   "source": [
    "import topmost\n",
    "from topmost.data import download_dataset\n",
    "\n",
    "device = \"cuda\" # or \"cpu\"\n",
    "dataset_dir = \"./datasets/NYT\"\n",
    "download_dataset('NYT', cache_path='./datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>train_size:  8254\n",
      "===>test_size:  918\n",
      "===>vocab_size:  10000\n",
      "===>average length: 175.429\n"
     ]
    }
   ],
   "source": [
    "# load a dataset\n",
    "dataset = topmost.data.BasicDatasetHandler(dataset_dir, read_labels=True, device=device, as_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▎         | 5/200 [00:49<30:01,  9.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 005 loss: 1745.447\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 10/200 [01:25<23:51,  7.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 010 loss: 1722.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 15/200 [02:08<27:53,  9.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 015 loss: 1726.942\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 20/200 [02:23<11:11,  3.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 020 loss: 1704.977\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▎        | 25/200 [02:35<07:43,  2.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 025 loss: 1684.670\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 30/200 [02:47<07:02,  2.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 030 loss: 1653.177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 35/200 [02:59<06:47,  2.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 035 loss: 1642.648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 40/200 [03:12<06:33,  2.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 040 loss: 1622.785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▎       | 45/200 [03:24<06:18,  2.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 045 loss: 1619.764\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 50/200 [03:36<06:07,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 050 loss: 1613.429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 55/200 [03:48<05:48,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 055 loss: 1594.577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 60/200 [04:09<11:35,  4.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 060 loss: 1590.442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▎      | 65/200 [05:24<29:21, 13.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 065 loss: 1571.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 70/200 [05:37<09:08,  4.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 070 loss: 1580.543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 75/200 [05:49<05:41,  2.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 075 loss: 1569.817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 80/200 [06:01<04:53,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 080 loss: 1558.638\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▎     | 85/200 [06:13<04:40,  2.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 085 loss: 1548.099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 90/200 [06:25<04:38,  2.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 090 loss: 1549.237\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 95/200 [06:42<05:34,  3.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 095 loss: 1540.831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 100/200 [06:58<05:32,  3.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 100 loss: 1540.966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▎    | 105/200 [07:15<05:12,  3.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 105 loss: 1530.688\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 110/200 [07:30<04:14,  2.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 110 loss: 1531.443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▊    | 115/200 [07:44<03:50,  2.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 115 loss: 1526.802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 120/200 [07:56<03:20,  2.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 120 loss: 1519.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 125/200 [08:09<03:03,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 125 loss: 1518.711\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 130/200 [08:21<02:50,  2.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 130 loss: 1521.188\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 135/200 [08:33<02:38,  2.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 135 loss: 1511.449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 140/200 [08:45<02:25,  2.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 140 loss: 1506.378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▎  | 145/200 [08:57<02:12,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 145 loss: 1502.324\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 150/200 [09:09<02:01,  2.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 150 loss: 1501.834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 155/200 [09:21<01:48,  2.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 155 loss: 1503.194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 160/200 [09:33<01:36,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 160 loss: 1500.108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▎ | 165/200 [09:45<01:25,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 165 loss: 1496.894\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 170/200 [09:58<01:13,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 170 loss: 1491.770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 175/200 [10:10<01:00,  2.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 175 loss: 1492.107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 180/200 [10:22<00:48,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 180 loss: 1488.774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▎| 185/200 [10:34<00:36,  2.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 185 loss: 1486.688\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 190/200 [10:46<00:24,  2.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 190 loss: 1484.077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 195/200 [10:58<00:12,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 195 loss: 1482.950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 200 loss: 1480.412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "# create a model\n",
    "# model = topmost.models.SawETM(vocab_size=dataset.vocab_size, num_topics_list=[10, 50, 200], device=device)\n",
    "# model = topmost.models.HyperMiner(vocab_size=dataset.vocab_size, num_topics_list=[10, 50, 200], device=device)\n",
    "model = topmost.models.TraCo(dataset.vocab_size, num_topics_list=[10, 50, 200])\n",
    "model = model.to(device)\n",
    "# create a trainer\n",
    "trainer = topmost.trainers.HierarchicalTrainer(model)\n",
    "\n",
    "# train the model\n",
    "trainer.train(dataset)\n",
    "\n",
    "# Or directly use fit_transform\n",
    "# top_words, train_theta = trainer.fit_transform(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= Layer: 0 number of topics: 10 =======\n",
      "Topic 0: lonely catholic feminist reign revered editorial politician diaspora equality intellectual newspaper bedrock stirring divorce meghan\n",
      "Topic 1: election vote senate republicans democrats primaries votes senator republican candidates endorsing president mcconnell candidate presidential\n",
      "Topic 2: growth earnings saudi climbed donors index stubbornly rates inflation bank bailey gains jobs yields increases\n",
      "Topic 3: troops turkey iran kuwait military buildup forces iraqi iranian iraq militias mediterranean zaporizhzhia eastern mykolaiv\n",
      "Topic 4: now just barbaro know archived michael recording work wow like yeah huh states really want\n",
      "Topic 5: scene punch festival red bull playing ball yards plate season seconds tall screen camera hip\n",
      "Topic 6: patients disease covid cdc prevention transmissible symptoms coronavirus dose positive infected intensive virus hospitalized diagnosis\n",
      "Topic 7: superintendent louisville probation minneapolis unarmed excessive ferguson firearms perpetrators concealed bullets gun incarcerated staten police\n",
      "Topic 8: billion consumers companies leases debt regulated profits cap subsidy annually partnerships trillion regulatory deposit pricing\n",
      "Topic 9: investigation lawyer complaint investigators counsel inquiry investigations testimony subpoena prosecutors witness documents wrongdoing misled cohen\n",
      "======= Layer: 1 number of topics: 50 =======\n",
      "Topic 0: committee bill rules call secretary order report conduct constitutional case ambassador security hearings pro witnesses\n",
      "Topic 1: surfside rodriguez champlain condo rubble survivors rescue searching bodies debris escaped belongings firefighters search towers\n",
      "Topic 2: water pollution battery climate drought cars plants emissions epa hurricane greenhouse vehicles coal trucks temperatures\n",
      "Topic 3: panel regulations pruitt rules obama privacy agency emissions rule telecommunications epa environmental regulatory regulation arbitration\n",
      "Topic 4: sarah mother jennifer story friends walking man woman daughter guys amazing love guy emotional applause\n",
      "Topic 5: inoculated variants ebola epidemiologists variant clinical hamas syndrome vaccinating outweighed newsom boosters contagious booster prescribed\n",
      "Topic 6: weinstein car hill floyd restaurant photos ton fake code hotel annie bystander filming committing arrest\n",
      "Topic 7: judge case court trial jury sexual lawyers epstein courtroom guilty charges victims prison jail sentence\n",
      "Topic 8: counterintelligence manafort page mueller rudolph comey special iii barr subpoenaed recused attorney fixer giuliani interference\n",
      "Topic 9: budget merger arab kingdom cuts ancient rousseff lebanon carriers egypt province overtime activists arrested airlines\n",
      "Topic 10: chinese china hong kong jinping taiwan beijing communist mainland media mao islands extradition propaganda liu\n",
      "Topic 11: russian russia putin ukrainian vladimir kremlin moscow kyiv soldiers sanctions navalny invasion zelensky crimea western\n",
      "Topic 12: data rate unemployment voters older poor polls adults higher poll eligible hispanic income voter badly\n",
      "Topic 13: data company report manufacturing agency employees regulators epa delivery johnson results classified said production protection\n",
      "Topic 14: art museum temple painting artist chef tourists manhattan trash restaurants arts gallery auction apartments estate\n",
      "Topic 15: app tech google apple facebook users silicon zuckerberg apps user musk game ads twitter uber\n",
      "Topic 16: mohammed kingdom prince bin khashoggi killing crown jamal suspects qaeda hails accusations consulate accused detained\n",
      "Topic 17: catalonia catalan india delhi oxygen independence barcelona modi migrants mosul rio indian lockdown separatist spain\n",
      "Topic 18: players games nfl tournament basketball league championship season fans tennis nba coach player stars baseball\n",
      "Topic 19: film documentary reading magic watching things motives listening movies headlines abstract independents theater movie reviews\n",
      "Topic 20: fox murdoch news stone nbc actor ailes roger movie podcast stepping actress cable harvey comedy\n",
      "Topic 21: cases spread outbreaks deaths restrictions wuhan texas hospitals quarantine protocols mask masks fined tracing health\n",
      "Topic 22: bill tax legislation congress cuts income spending taxes budget repeal poor money wealthy senator mnuchin\n",
      "Topic 23: korea missiles north nuclear korean missile kim pentagon iran launch escalate iranians ballistic launched sanctions\n",
      "Topic 24: security terrorist qaeda islamic terrorism counterterrorism torture muslim isis ban airport countries pentagon intelligence snowden\n",
      "Topic 25: union health secretary trade partnership education businesses moral unions order company contracts corporate contract andrew\n",
      "Topic 26: redistricting legislature abortions ballot abortion supreme rights roe governor voting texas court voter ballots map\n",
      "Topic 27: royal queen diana father prince palace actress boy art writer book married ballet harry writers\n",
      "Topic 28: prime minister parliament european britain brexit johnson bloc union exit parliamentary boris netanyahu europe parties\n",
      "Topic 29: residents school york county angeles fort smith housing community construction jose credit refugees asian mayor\n",
      "Topic 30: sanders super voters warren biden bernie primary nominee party cruz polls iowa buttigieg candidacy nomination\n",
      "Topic 31: insurance rent moratorium eviction tenants medicaid affordable mandate states opioid care renters health rental landlords\n",
      "Topic 32: israeli israel shiite insurgents gaza afghan kabul fighters militants airstrikes palestinians palestinian rebel taliban jerusalem\n",
      "Topic 33: taliban fed confirmation afghanistan nominated rousseff afghan dilma nominee petrobras cabinet deputy delicate reserve abdullah\n",
      "Topic 34: hindu confederate jewish jews muslim muslims flag haley rally gay nationalist modi orthodox remove ultra\n",
      "Topic 35: reporters jan rallies house observers president endorse participating capitol causes marches representative rely journalists trump\n",
      "Topic 36: countries imports exports trade european europe canada russia gas flows natural germany oil britain india\n",
      "Topic 37: electoral results georgia ballots runoff vote fraud won absentee election generic winner race overturn certified\n",
      "Topic 38: antibodies immune virologist hepatitis virus treatments drug cells diseases genetic cancer experimental treatment scientists diabetes\n",
      "Topic 39: facebook said employees learn students sources education teachers motivation anonymous school wrote members endorse content\n",
      "Topic 40: gov governor cuomo albany commission ethics democrat nixon race york silver mayor progressive primary harassment\n",
      "Topic 41: protesters protests videos gunman tear officer arrested protest riot capitol shot shooting video violent violence\n",
      "Topic 42: killing officials tourist miles car moderate smoke madrid taxi backup chat cover killed guard sticking\n",
      "Topic 43: american violence east border united joe burns andrew won economy climate house streets york suleimani\n",
      "Topic 44: nato vice macron tokyo olympics olympic president meeting tayyip alliance ceremony athletes francis recep games\n",
      "Topic 45: shareholders investment fund funds settlement sovereign equity hedge musk stake company billion money shares securities\n",
      "Topic 46: investors prices stock economy fed markets bond bonds recession market economists rates financial stocks volatility\n",
      "Topic 47: subscribers skating toe jumps medal sox netflix quarter ratings triple disney gold cable halfway rep\n",
      "Topic 48: bitcoin robots smart want ups reader milk machines maybe love rough mom meat fun things\n",
      "Topic 49: students parents college schools construction children women admissions school rural harvard jobs remotely colleges hispanic\n",
      "======= Layer: 2 number of topics: 200 =======\n",
      "Topic 0: checkpoint shelling credit random correctional prisons deaths children detonated disposal constant airstrike swat terminal idaho\n",
      "Topic 1: nord der lithuania tariffs pipeline von baltic iphones tariff germany countries sweden bloc aluminum glasgow\n",
      "Topic 2: race raphael kemp warnock recount ossoff defeats doug electors ins vying jon maloney collins pollsters\n",
      "Topic 3: margaret queue enslaved elegant recounts favorites workings reviews classical accent thriller kennedy essays print reading\n",
      "Topic 4: icu gloves hood mutations item upgraded pneumonia wuhan protein certificates viruses levine matt nursing soap\n",
      "Topic 5: report committee neil clinton special subjects obtain committees nixon mentioned schiff actual phase damaging house\n",
      "Topic 6: overdose evictions overdoses fentanyl debts pills exchanges unpaid homelessness obamacare insurers premiums penalties marketed licensed\n",
      "Topic 7: women children admissions skilled jobs sectors studies disabilities employment mothers workers rural construction students retail\n",
      "Topic 8: herald charleston ferguson abused blacks gunfire lee robinson chased bible laugh anderson policing travis correctional\n",
      "Topic 9: hedge loans partner capitalize managing company creek shrink oversees amazon teamed settlements managers dollars estate\n",
      "Topic 10: party compromise elections wing promote boris parties grace britain parliament pro minister cameron favors parliamentary\n",
      "Topic 11: gas solar offline gases pipelines nord dioxide grid rainfall methodology figuring culturally improvements forests electric\n",
      "Topic 12: estate lots renovation manhattan neighborhood temple penn eat carve houses streets newark food tag tourists\n",
      "Topic 13: playoffs goodell yankees warriors hockey players season athlete griner locker brady finals coaching espn anthem\n",
      "Topic 14: skin twists marco want ineffective obamacare doubles punching honorable grandfather bed hopeful punch honestly machinery\n",
      "Topic 15: gerrymandering discriminatory rights restrictive kemp legislatures incumbents narrower manchin restrict abortion filibuster flipping elections election\n",
      "Topic 16: rajoy mariano decree suspend warsaw separatists champions polish orderly pedro migrants independence retain asylum manchester\n",
      "Topic 17: patricia nuanced recognition prevailing underage dimension lighter recognizes negotiated feels predatory trafficking story agreement sort\n",
      "Topic 18: stereotypes divorced dress loving sympathy oscar dad soul horns doubles films hat excitement affection dream\n",
      "Topic 19: sheldon silver fifa tournaments kickbacks financier bribery disgraced silva testified dealings commission preet subsidiary mit\n",
      "Topic 20: girl walls birth married remembers chapel royal boy forever precious seated remembered story horrific amateur\n",
      "Topic 21: russian influence gone posts twitter borders operates ukrainian videos account campaigns hype allegedly caught indicted\n",
      "Topic 22: withdrawal royal johnson routine matthew unlock duty stepping volunteer queen palace trap union exit regret\n",
      "Topic 23: screening security jihadist isil libya attacker iraqis terrorism iraq airport terminal sudan saddam american militias\n",
      "Topic 24: kashmir barcelona restive graft anniversary rebellion jailed demonstrators carrie fifa yellow ouster unilaterally aleksei oxygen\n",
      "Topic 25: rupert fox anchor defamation resigns slur comedy cable editor journalism rapper rated magazine nbc roger\n",
      "Topic 26: committee house justice devin interfere chairmen officials judiciary nsa investigative nadler subpoena watchdog appointees intelligence\n",
      "Topic 27: finalized wireless arbitration broadband antitrust enrichment sprint deregulation vienna leagues revoke neutrality regulations rules mlb\n",
      "Topic 28: lightfoot assignment rob lori locals mode throat dining slip classmates cafe covid cruise sars respiratory\n",
      "Topic 29: mulvaney giuliani reconstructed inquiry special president attorney bidens lawyer blower zelensky mick mueller counsel trump\n",
      "Topic 30: danish oxygen india surpassed pakistan singapore plunge egypt king pilgrims thailand brazil indian easter globally\n",
      "Topic 31: aids productive fun suggestions learn brand character cook stuff machine digital habit hill realized design\n",
      "Topic 32: queue starring favorites recounts featuring editor commentary essays reviews journalism film enjoying subscriber classical enslaved\n",
      "Topic 33: birds pumping gently plug sustainability hybrid sand cheaper lobby farmer bird gavin cattle store excellent\n",
      "Topic 34: white lagged percent successive southwest utterly latinos sugar clinton notes analyzed shift arguably des lake\n",
      "Topic 35: portland charlie kazakhstan chapters prosecutions slaughter neo agents charred whatsapp xinjiang militia aleksei deployed extremism\n",
      "Topic 36: money joe recording trump bernie archived ben president burns productive graduation hmm mitigate warren unemployment\n",
      "Topic 37: ukraine devin barbaro house impeached erdogan republicans kent hearings ambassador trump meeting michael secretary sondland\n",
      "Topic 38: restive pakistan autonomy bashar assad mosque ruthless factions cleric coup belarus ouster sectarian rabbi syrian\n",
      "Topic 39: peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\n",
      "Topic 40: commission legalize amtrak congestion staffers walt tony mayor blasio sara silver cuomo hamilton assemblyman sheldon\n",
      "Topic 41: confirmation renew yellen marshal chuck nominated signaled secretary chairman speculation powell roy chosen governors casting\n",
      "Topic 42: american assad missiles location russians phase pentagon responsibility united airstrikes weapon ukraine operation weapons vehicle\n",
      "Topic 43: gender children teens students pregnant wear deliberate training student school nurses society teenage babies child\n",
      "Topic 44: church flag instinct practically theory community man crews shoot shot tomorrow just applause crime knock\n",
      "Topic 45: mnuchin panel boehner congress postal nadler recess nsa postmaster senate borrowing obama appeals avert pruitt\n",
      "Topic 46: ethnic setbacks coalition maintained fierce blackout sums vying caps hub funded steered stave departures operates\n",
      "Topic 47: saudi activists rulers arabia clerics prophet ruler anticorruption dissidents kingdom evasion sheikh duchess insults financier\n",
      "Topic 48: percent upshot analytics vote rating respondents survey statistical median surveys cusp exclude rican margin delegate\n",
      "Topic 49: bidens sergey kyiv putin officials gordon rudy nicholas taylor sondland ukrainian zelensky misled informing schmidt\n",
      "Topic 50: siena voters electorate hispanic unfavorable incarcerated hispanics pollsters poll error sample partisanship mrs impression southeast\n",
      "Topic 51: talent howard fox literary rupert comedy intercontinental nbc insider channels celebrity boasting theaters succession valued\n",
      "Topic 52: redacted fbi page subpoenas comey unsealed dept laundering inventory prosecutors surveillance records firing secretly defects\n",
      "Topic 53: stigma unvaccinated herd frustrating ventilation like poor occurring grandchildren kid bit really work guiding care\n",
      "Topic 54: city lease ferry homeless midtown commuters exhibition yorkers dancing acre jewelry plaza art homelessness posters\n",
      "Topic 55: lesser party democratic sugar won income agriculture billionaires cory finishing beto wright switched sanders burns\n",
      "Topic 56: upscale suburban suburbs violence epstein themes cities strongholds disperse uptick reid neighborhood ballot staten culturally\n",
      "Topic 57: gillibrand nbc bros news bannon fox anchor gabriel bullying slate cnn harassment poster kirsten rated\n",
      "Topic 58: wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\n",
      "Topic 59: landslide city letitia offenders accomplishments challenger bharara cuomo mansion primary citywide eric chairs positioned offenses\n",
      "Topic 60: bishop priests cardinal duchess roman church sexually metoo abuse honors dating otto catholic denied priest\n",
      "Topic 61: sinema deceptive loophole cuomo corporations proposition raiser prop contributions tax legalize proposes legalization initiatives campaign\n",
      "Topic 62: data methodology emails registration cybersecurity assessed methods incorrect servers trove carrier adverse databases warrant subpoenas\n",
      "Topic 63: police shells shelling breached qanon frenzied simultaneous animals east mortar battling metro siege insurrection rocket\n",
      "Topic 64: bribes khan silva funnel dictatorship anticorruption province carriers schemes designation leftist rousseff frontier lender overtime\n",
      "Topic 65: injections execution executions injection lethal oklahoma oxycontin trial defendant patent botched drugs opioid samsung corrections\n",
      "Topic 66: mueller special gates accusations allegations markle agent wikileaks email contacts murdoch formerly fake cambridge guilty\n",
      "Topic 67: kidnapping sgt assailants prophet killers kidnapped muhammad hostages nigerian abducted gunned hostage gunmen carnage mosque\n",
      "Topic 68: immigrant stanley ban vital order permanent employees refuse qualified unauthorized workers officials deaths statistics seven\n",
      "Topic 69: publishers hashtag semitic censorship islam ethnicity pioneer goldberg flagged orientation sympathy nyu pundits discourse audiences\n",
      "Topic 70: nuclear iran atomic warmbier uranium tehran korea vessels otto ships north philippines hawks cyberattacks cyber\n",
      "Topic 71: mosul dam kurdish villages river ruins metropolis soar floods mykolaiv miles rainfall acres retake towns\n",
      "Topic 72: biden donation race caucuses poll rubio sanders diminish trailing iowa campaign cruz vermont polling rodham\n",
      "Topic 73: turkish militants monitor claimed fighters chambers tasked shiite controlled attempting backlash gathered measures arrests courthouse\n",
      "Topic 74: rubber protesters supremacists demonstrators slogans protests clashes goodell disperse violent inaction demonstrations lam tear police\n",
      "Topic 75: repercussions recommendations prevented opioid mentally pending weighs agree trumps directive mandate treating barring doctor statement\n",
      "Topic 76: suspension fda afghan ghani ashraf biontech panel shortfall afghanistan coordinator provincial abruptly positive cognitive fed\n",
      "Topic 77: rajoy socialists socialist deadlock pedro inconclusive spain mariano abdullah referendum plunged ghani centrist stalemate impartial\n",
      "Topic 78: pen delegate bend delegates jean des pour caucuses nomination ideologically hat passionate ascent mate pete\n",
      "Topic 79: loretta supported removal remove haley supremacists nationalists clashing gop congresswoman sworn flag kemp lynch takeaways\n",
      "Topic 80: meghan navalny kyiv soldiers zelensky soldier dmitri western blinken contingent ukrainian cold intervention wuhan military\n",
      "Topic 81: said employees work resort anonymous employers members education classmates labor credit learn motivation permanent unions\n",
      "Topic 82: care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\n",
      "Topic 83: gatherings redacted extension johnson downing british servant gray deadline continental mrs liz parliament defy showdown\n",
      "Topic 84: totalitarian openness censors communist echelons reign usher handover china prepares elites chinese jinping ying japan\n",
      "Topic 85: cuomo served graft sullivan politico dept amounted tenure denied banks ashraf holder prosecutor retire abruptly\n",
      "Topic 86: tax manchin filibuster sinema glasgow progressives provisions reconciliation liberals democracies majorities drilling bipartisan bridges ceiling\n",
      "Topic 87: soldiers torture propaganda besieged palestinian eastern border terror proclaimed village occupation getty collective east grim\n",
      "Topic 88: stone meeting jared flynn bannon roger accusations campaign kremlin son paul kushner mrs matthew wikileaks\n",
      "Topic 89: frontier bribes musicians designation blessing vision antigovernment branch pushback tourism nyu insurgency uae restored hezbollah\n",
      "Topic 90: pollsters sampling poll probability polls logic consumer swung positioned gauge alpha earnings increases explanations takeaways\n",
      "Topic 91: water sirens tornadoes residents tornado farms riders busiest generators sidewalks flushing coverings caribbean factories quarantines\n",
      "Topic 92: spells pediatric spark tales cope prevalent depleted toys furniture exceed scarce sustainable goods chunk partial\n",
      "Topic 93: poker lineup strengths receptive button quest swinging netflix smartphone smart justin plug shifts franchise motion\n",
      "Topic 94: okla wis ariz ark ala minn fla ind mich conn del tenn utah colo ore\n",
      "Topic 95: lists connect writer laughed acquiring college wrote tip guided email woman weinstein displayed fun instagram\n",
      "Topic 96: royals poet harry poetry slave sussex acclaimed fictional duchess windsor meghan literature markle novels prose\n",
      "Topic 97: architecture skies singers sung gala opera hardware songs civilization columns pierre paintings productions embraces travels\n",
      "Topic 98: acquisition anthem clubs leagues brands annually format richest jpmorgan rupert super cents mlb premier acquire\n",
      "Topic 99: suleimani flexible benghazi accompanied iraq turf muscular convoy exploding mourning nearing andrew backfire dilemma elderly\n",
      "Topic 100: slowing tesla automaker maker deliveries disruptions markedly quarter constraints samsung motor quarterly curve sales slump\n",
      "Topic 101: wobble rout strings productivity bitcoin acceptance darkness programmers navigating weathering robots marketplace tasks enabling groceries\n",
      "Topic 102: applause bell kyle grace los latino instruction educators descent reservoir chicago latinos congregation asian residents\n",
      "Topic 103: deutsche softbank loan trades pension subsidiary bankers sec loeffler stanley mortgages lenders fargo goldman windfall\n",
      "Topic 104: arabia saudi abbas mattis ghani bahrain reassure arab salman boon boehner qatar persian tillerson pave\n",
      "Topic 105: discourse spends studying historian divorce societies essay nigeria slaughter outfits neglected jonathan mothers halloween inclusion\n",
      "Topic 106: maggie cure archived recording dictate pharma order care postmaster redfield union meaningful safest katie workers\n",
      "Topic 107: vary stream detonated steering meters broadcasting reliance persist monthly reliably terminated thick gas uneven lighter\n",
      "Topic 108: city temple roosevelt pedestrians lease enclave artist wine midtown halloween plaza planners foot subway graffiti\n",
      "Topic 109: constitutionality supreme justices vacancies upheld firearms violates ban regulate unconstitutional injunction cody rulings evictions localities\n",
      "Topic 110: aleksei navalny siberian putin vladimir mykolaiv moscow russia sergei blackout griner nerve poisoning dmitri mourners\n",
      "Topic 111: corroborate members said anonymous satisfied women motivation sources wen education employees learn employers reliable fulfilling\n",
      "Topic 112: extremist terrorist antigovernment guard white counterterrorism capitol missions maj readiness afghanistan paramilitary raids indications shortcomings\n",
      "Topic 113: oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\n",
      "Topic 114: athletes summit dialogue tillerson olympics jae tokyo moon diplomatic winter athens rex pop north pyongyang\n",
      "Topic 115: donation biden tariffs tillerson kenya equitable meeting president pence incited vice blinken supremacists democracies thanked\n",
      "Topic 116: japan base medal consecutive wolf podium episode seventh jae lineup artistic sox olympic broadcast score\n",
      "Topic 117: aleppo isil raqqa kunduz damascus warplanes mykolaiv nigerian mosul haram bombers provincial siberian islamist civilians\n",
      "Topic 118: transgender rebukes gay equality prohibit condemns greene condemn fringe hook discriminatory amendment orientation loretta carolina\n",
      "Topic 119: nepal earthquakes cathedral frantic asleep earthquake stampede tornado nasa tropical tornadoes messenger ripped blaze smashed\n",
      "Topic 120: electors trump hillary mccain mar contests clinton democratic trails masters race gore comfortably christie desantis\n",
      "Topic 121: doctrine court warrant wilson cellphone printed privacy judge affirmative cody sued violating consent unlawful justices\n",
      "Topic 122: cases omicron transmission testing indoor tested caseloads hospitalizations infections quarantines masking gyms transmissible positivity mask\n",
      "Topic 123: ecosystem scott jason sydney wildlife susan vacation southeastern morrison species property bizarre soap trees surreal\n",
      "Topic 124: warmbier missile coma yemeni korea yemen airstrike poisoning otto jong north crown kim korean honors\n",
      "Topic 125: scalise campaign mulvaney candidates ostensibly candidate mccain journalists qanon cover cheney mcconnell supremacists maloney mccarthy\n",
      "Topic 126: citywide york holder cuomo emanuel mayoral upstate adams city symbols mayor ralph assemblyman hochul charleston\n",
      "Topic 127: knife mcdonald gregory manslaughter pistol cameras pinned shootings swinging toronto protester crashes rifles police fatal\n",
      "Topic 128: authoritarian conceal canadians transmissible chinese dynamics import soviet undertaking now batch portraying dynamic mobilize reconcile\n",
      "Topic 129: silva swiss graft evangelical brazilians michel shareholder acquire powerhouse brazil equity fifa stake deutsche alibaba\n",
      "Topic 130: patients medication detect strokes drugs infants cognitive symptoms pill chronic transmit drug muscle consume tests\n",
      "Topic 131: sing writing really comic obsessed lifetime write exciting puzzle belongs beautiful rehearsal storytelling recounts seed\n",
      "Topic 132: unifying meeting elect treats vice oval olympics olympic dear president comments presidency erdogan macron ivanka\n",
      "Topic 133: costumes halloween yale immunized rating pediatrics abuse olds lgbtq couples mit children fetus university profoundly\n",
      "Topic 134: shareholder tabloid hacking transaction publishers bros harassed elon ailes max famously slate bezos murdoch cambridge\n",
      "Topic 135: nominate powell accusations rousseff meddling jerome graft manipulate ashraf cabinet dilma sisi ouster remarks chairwoman\n",
      "Topic 136: merger airline carrier airlines recipients routes ban mentally exempt destinations concentrated carriers exclusively licenses valid\n",
      "Topic 137: insurrection murkowski convict cheney keepers certification stormed wyoming romney boys mitt incited greene ardent proud\n",
      "Topic 138: brexit austerity socialists hollande continent warsaw polish euro rajoy athens merkel humiliating french macedonia membership\n",
      "Topic 139: brad articles vote senate house impeach partisanship election julie president impeached pelosi appointing votes postmaster\n",
      "Topic 140: ambulance certifying gunshots bryan filming whereabouts minutes steal pavement homicide footage reinforcements gunshot wheel keepers\n",
      "Topic 141: veterans speeches vice president jared members campaign kushner values reporters supporting meeting speech barack phrase\n",
      "Topic 142: van entrance switches pacing flies intercept spreads clothes taxi autopsy joking ruthless pulls driveway cairo\n",
      "Topic 143: morale drone soldiers drones occupied atrocities offensive zaporizhzhia volodymyr telegram yulia soldier surrender humanitarian commander\n",
      "Topic 144: motors inland automakers volkswagen balloons electric motor amtrak automaker batteries ford water wheeler autonomous diesel\n",
      "Topic 145: assailants gunmen wounding shootout allegiance compound bernardino stabbed injuring rampage detonated hallway plowed bombings gunshots\n",
      "Topic 146: enriched capability nuclear inspectors terminated atomic tehran collapses tiktok iranian produce treaty huawei ceased inspections\n",
      "Topic 147: intelligence espionage ransomware adversary unspecified anonymous detainee huawei arrested interrogation secrets dissidents pakistani hostage poisoned\n",
      "Topic 148: midterms stampede tornado pilots northeastern siberian nepal evacuation probe cathedral diamond notre bull concert quarantined\n",
      "Topic 149: india indians muslims roman secular fetus liberty religion ultra narendra ethnicity transgender exemption murders religious\n",
      "Topic 150: soaring contraction worshipers shrank deepening britons surpassed economy squeezing gross slump drag australians continent downturn\n",
      "Topic 151: irs moderate poor inflation spells tangible purchasing budgets spell affect tomorrow fared chunk apparel toys\n",
      "Topic 152: pharmacies overdose opioids pharmacy fentanyl municipalities settlement overdoses oxycontin licensed premium addiction drug tribes manufacturer\n",
      "Topic 153: kaiser locked gunman butler christmas rubble closed distancing suddenly outside chaotic heart specialists curtain horses\n",
      "Topic 154: gillibrand harris hampshire kirsten warren mate sanders mayor bend caucuses iowa vermont senator campaign democratic\n",
      "Topic 155: genuine convincing reassuring staring tapes anderson mine proves cheering eyed convinced convince valid raw knees\n",
      "Topic 156: queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\n",
      "Topic 157: snapchat networking mayer verizon kalanick yahoo elon alibaba musk hailing gaming subscription advertisers tiktok cave\n",
      "Topic 158: participants baltimore bleeding trial results trials johnson moderna warp peer incorrect conclusion reviewed company report\n",
      "Topic 159: said reporters members staff white money rallies representative ads stormy causes desantis campaign observers requests\n",
      "Topic 160: manufactured sources distributing india recommendation supplies officials rollout chains raw astrazeneca relying databases freight shortages\n",
      "Topic 161: slide seas totaled screens samsung irma revised dam flooded proclaimed carmaker ford merger inches pitched\n",
      "Topic 162: attorney centered defendant incite felonies guilty lawyers resignations justice charges vacancies acquitted indictment obstruction sheldon\n",
      "Topic 163: extortion subversion assemblyman misdemeanor catholics convicted sheldon towering courthouse indictment financier silver bribery sentenced prominence\n",
      "Topic 164: unanswered royalty bryant actress women flowers abuse oral assaulted metoo rafael jane uncle keith cincinnati\n",
      "Topic 165: holocaust rabbi charlottesville lesbian nationalism neo migrants secular nikki lgbtq haley rally stricter condemns massacre\n",
      "Topic 166: barrels oil atrocities ports mediterranean military ocean persian uae reactor deficits exercises sullivan invading reliant\n",
      "Topic 167: car store fake undercover restaurant asks handgun code videos departments murder photos photo app parking\n",
      "Topic 168: chokehold homicide manslaughter officers mcdonald officer indict minneapolis unarmed derek aggravated offenses misdemeanor handcuffed jurors\n",
      "Topic 169: border countries profitable immigrants migration greed union attractive reductions ross poorer corporations andrew united realization\n",
      "Topic 170: stone ing julian campaign clintons trump publicity informal tricks wikileaks adviser roger writes constitutes insight\n",
      "Topic 171: aviv tel naftali isaac benjamin abbas likud annex diaspora annexation netanyahu inconclusive centrist minister jerusalem\n",
      "Topic 172: facial android extract companies servers data neutrality technology antitrust iphones ransomware computing ibm tiktok click\n",
      "Topic 173: vaccine doses pfizer measles astrazeneca immunity biontech virus vaccination vaccines vaccinated cdc variant dose infection\n",
      "Topic 174: gym uncommon bearing noise photographers atop boom rented avenue colored swimming volunteers tightly roof invaded\n",
      "Topic 175: ashley hesitant violent problematic pneumonia spread solved case spiral breathing situations mail assure violation restraint\n",
      "Topic 176: infuriated commanding endorsement cohn deprived decisive wins impeachment caucuses unify mate primaries alex negotiator delegates\n",
      "Topic 177: broadway acclaimed performer soundtrack poetry phillips musical tony album musician opera singer sang awards song\n",
      "Topic 178: forces retake iraq iraqi captive explosives oct amnesty baghdad monitor base afghans prioritized troops controlled\n",
      "Topic 179: buyers advise modified certificate additional handling intersection discretion dealer housing storage safe effective recovered fearing\n",
      "Topic 180: grocery retailer trade hybrid programmers companies oriented currency globally brands corporate incentive adaptation tariff retailers\n",
      "Topic 181: reopen historically rural nationwide speed college community nowhere california nightmare indiana colleges abolish harvard lgbtq\n",
      "Topic 182: rioters steal mattis pompeo reinforcements generals militia siege rationale cooper maggie impeachment trump qassim launch\n",
      "Topic 183: party ministries ghani win portugal consecutive polling colombia voters won hinder sara parliamentary prime minister\n",
      "Topic 184: nightclub arabic fence hiding pulse freddie exchanged ensued hole gunman von boko singh balls infant\n",
      "Topic 185: schwartz schmidt bell archived story amy blake yelling recording sarah yeah wow cooper julie kate\n",
      "Topic 186: reports timed approximately resolving strongest dates compares missing rolling ore instances calif miss wash informing\n",
      "Topic 187: mutations princess dumped viruses protein dec seattle identical munich cascade collected samples shanghai victoria wuhan\n",
      "Topic 188: abu houthi hezbollah sectarian houthis sana sunni mansour lebanese bahrain insurgency militias qatar yemeni afghans\n",
      "Topic 189: abu superintendent gavin storied residents workers blankets providence collar rents refugee sacramento boko diego mill\n",
      "Topic 190: eligible canada united adult older goal optimism residents mandates hurdle populations pediatric average breakthrough shots\n",
      "Topic 191: unconscious goodman overwhelmed collapses sacrifice storm mob update sooner partially handgun kentucky floor reinforced knock\n",
      "Topic 192: said report credit employees data workers erin supervisors reporting sources york labor businesses expects department\n",
      "Topic 193: hackers confidential disney spying studio nick emails hacker computers cybersecurity communicating traces distributing server disclosing\n",
      "Topic 194: percent yields economic mortgage slowdown outlook stimulus mortgages layoffs bank lending forecasts contraction jobless borrowers\n",
      "Topic 195: appeals circuit abortion court proposition pregnancies convictions unconstitutional rights orleans abortions injunction enforcing kavanaugh justices\n",
      "Topic 196: repairs owed developer collectors devised master fee turnover alternatives preservation inspections leases branches prestigious rewrite\n",
      "Topic 197: retail businesses salary labor tenth workers unions corporate cash loans spending firms work transportation credit\n",
      "Topic 198: gang synagogue scholarship readers enslaved rendered prose slavery massacre essays provocative jesus supremacy whites beatings\n",
      "Topic 199: slam seeded djokovic match cup spectators ball scored matches scoring novak champions finals singles sneakers\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xiaobao/local/miniconda3/envs/topmost/lib/python3.8/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TD: 0.8016666666666666\n",
      "{'Purity': 0.6089324618736383, 'NMI': 0.32137661515027016}\n",
      "{'acc': 0.7233115468409586, 'macro-F1': 0.5320532658809204}\n"
     ]
    }
   ],
   "source": [
    "########################### Evaluate ####################################\n",
    "import json\n",
    "import numpy as np\n",
    "from topmost import evaluations\n",
    "\n",
    "# get top words of topics\n",
    "top_words = trainer.export_top_words(dataset.vocab)\n",
    "# get theta (doc-topic distributions)\n",
    "train_theta, test_theta = trainer.export_theta(dataset)\n",
    "\n",
    "# compute topic coherence\n",
    "# refer to https://github.com/BobXWu/ECRTM\n",
    "\n",
    "# compute topic diversity\n",
    "TD = evaluations.multiaspect_topic_diversity(top_words)\n",
    "print(f\"TD: {TD}\")\n",
    "\n",
    "# evaluate clustering\n",
    "results = evaluations.hierarchical_clustering(test_theta, dataset.test_labels)\n",
    "print(dict(results))\n",
    "\n",
    "# evaluate classification\n",
    "results = evaluations.hierarchical_classification(train_theta, test_theta, dataset.train_labels, dataset.test_labels)\n",
    "print(dict(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= Layer: 0 number of topics: 10 =======\n",
      "Topic 0: lonely catholic feminist reign revered editorial politician diaspora equality intellectual newspaper bedrock stirring divorce meghan\n",
      "Topic 1: election vote senate republicans democrats primaries votes senator republican candidates endorsing president mcconnell candidate presidential\n",
      "Topic 2: growth earnings saudi climbed donors index stubbornly rates inflation bank bailey gains jobs yields increases\n",
      "Topic 3: troops turkey iran kuwait military buildup forces iraqi iranian iraq militias mediterranean zaporizhzhia eastern mykolaiv\n",
      "Topic 4: now just barbaro know archived michael recording work wow like yeah huh states really want\n",
      "Topic 5: scene punch festival red bull playing ball yards plate season seconds tall screen camera hip\n",
      "Topic 6: patients disease covid cdc prevention transmissible symptoms coronavirus dose positive infected intensive virus hospitalized diagnosis\n",
      "Topic 7: superintendent louisville probation minneapolis unarmed excessive ferguson firearms perpetrators concealed bullets gun incarcerated staten police\n",
      "Topic 8: billion consumers companies leases debt regulated profits cap subsidy annually partnerships trillion regulatory deposit pricing\n",
      "Topic 9: investigation lawyer complaint investigators counsel inquiry investigations testimony subpoena prosecutors witness documents wrongdoing misled cohen\n",
      "======= Layer: 1 number of topics: 50 =======\n",
      "Topic 0: committee bill rules call secretary order report conduct constitutional case ambassador security hearings pro witnesses\n",
      "Topic 1: surfside rodriguez champlain condo rubble survivors rescue searching bodies debris escaped belongings firefighters search towers\n",
      "Topic 2: water pollution battery climate drought cars plants emissions epa hurricane greenhouse vehicles coal trucks temperatures\n",
      "Topic 3: panel regulations pruitt rules obama privacy agency emissions rule telecommunications epa environmental regulatory regulation arbitration\n",
      "Topic 4: sarah mother jennifer story friends walking man woman daughter guys amazing love guy emotional applause\n",
      "Topic 5: inoculated variants ebola epidemiologists variant clinical hamas syndrome vaccinating outweighed newsom boosters contagious booster prescribed\n",
      "Topic 6: weinstein car hill floyd restaurant photos ton fake code hotel annie bystander filming committing arrest\n",
      "Topic 7: judge case court trial jury sexual lawyers epstein courtroom guilty charges victims prison jail sentence\n",
      "Topic 8: counterintelligence manafort page mueller rudolph comey special iii barr subpoenaed recused attorney fixer giuliani interference\n",
      "Topic 9: budget merger arab kingdom cuts ancient rousseff lebanon carriers egypt province overtime activists arrested airlines\n",
      "Topic 10: chinese china hong kong jinping taiwan beijing communist mainland media mao islands extradition propaganda liu\n",
      "Topic 11: russian russia putin ukrainian vladimir kremlin moscow kyiv soldiers sanctions navalny invasion zelensky crimea western\n",
      "Topic 12: data rate unemployment voters older poor polls adults higher poll eligible hispanic income voter badly\n",
      "Topic 13: data company report manufacturing agency employees regulators epa delivery johnson results classified said production protection\n",
      "Topic 14: art museum temple painting artist chef tourists manhattan trash restaurants arts gallery auction apartments estate\n",
      "Topic 15: app tech google apple facebook users silicon zuckerberg apps user musk game ads twitter uber\n",
      "Topic 16: mohammed kingdom prince bin khashoggi killing crown jamal suspects qaeda hails accusations consulate accused detained\n",
      "Topic 17: catalonia catalan india delhi oxygen independence barcelona modi migrants mosul rio indian lockdown separatist spain\n",
      "Topic 18: players games nfl tournament basketball league championship season fans tennis nba coach player stars baseball\n",
      "Topic 19: film documentary reading magic watching things motives listening movies headlines abstract independents theater movie reviews\n",
      "Topic 20: fox murdoch news stone nbc actor ailes roger movie podcast stepping actress cable harvey comedy\n",
      "Topic 21: cases spread outbreaks deaths restrictions wuhan texas hospitals quarantine protocols mask masks fined tracing health\n",
      "Topic 22: bill tax legislation congress cuts income spending taxes budget repeal poor money wealthy senator mnuchin\n",
      "Topic 23: korea missiles north nuclear korean missile kim pentagon iran launch escalate iranians ballistic launched sanctions\n",
      "Topic 24: security terrorist qaeda islamic terrorism counterterrorism torture muslim isis ban airport countries pentagon intelligence snowden\n",
      "Topic 25: union health secretary trade partnership education businesses moral unions order company contracts corporate contract andrew\n",
      "Topic 26: redistricting legislature abortions ballot abortion supreme rights roe governor voting texas court voter ballots map\n",
      "Topic 27: royal queen diana father prince palace actress boy art writer book married ballet harry writers\n",
      "Topic 28: prime minister parliament european britain brexit johnson bloc union exit parliamentary boris netanyahu europe parties\n",
      "Topic 29: residents school york county angeles fort smith housing community construction jose credit refugees asian mayor\n",
      "Topic 30: sanders super voters warren biden bernie primary nominee party cruz polls iowa buttigieg candidacy nomination\n",
      "Topic 31: insurance rent moratorium eviction tenants medicaid affordable mandate states opioid care renters health rental landlords\n",
      "Topic 32: israeli israel shiite insurgents gaza afghan kabul fighters militants airstrikes palestinians palestinian rebel taliban jerusalem\n",
      "Topic 33: taliban fed confirmation afghanistan nominated rousseff afghan dilma nominee petrobras cabinet deputy delicate reserve abdullah\n",
      "Topic 34: hindu confederate jewish jews muslim muslims flag haley rally gay nationalist modi orthodox remove ultra\n",
      "Topic 35: reporters jan rallies house observers president endorse participating capitol causes marches representative rely journalists trump\n",
      "Topic 36: countries imports exports trade european europe canada russia gas flows natural germany oil britain india\n",
      "Topic 37: electoral results georgia ballots runoff vote fraud won absentee election generic winner race overturn certified\n",
      "Topic 38: antibodies immune virologist hepatitis virus treatments drug cells diseases genetic cancer experimental treatment scientists diabetes\n",
      "Topic 39: facebook said employees learn students sources education teachers motivation anonymous school wrote members endorse content\n",
      "Topic 40: gov governor cuomo albany commission ethics democrat nixon race york silver mayor progressive primary harassment\n",
      "Topic 41: protesters protests videos gunman tear officer arrested protest riot capitol shot shooting video violent violence\n",
      "Topic 42: killing officials tourist miles car moderate smoke madrid taxi backup chat cover killed guard sticking\n",
      "Topic 43: american violence east border united joe burns andrew won economy climate house streets york suleimani\n",
      "Topic 44: nato vice macron tokyo olympics olympic president meeting tayyip alliance ceremony athletes francis recep games\n",
      "Topic 45: shareholders investment fund funds settlement sovereign equity hedge musk stake company billion money shares securities\n",
      "Topic 46: investors prices stock economy fed markets bond bonds recession market economists rates financial stocks volatility\n",
      "Topic 47: subscribers skating toe jumps medal sox netflix quarter ratings triple disney gold cable halfway rep\n",
      "Topic 48: bitcoin robots smart want ups reader milk machines maybe love rough mom meat fun things\n",
      "Topic 49: students parents college schools construction children women admissions school rural harvard jobs remotely colleges hispanic\n",
      "======= Layer: 2 number of topics: 200 =======\n",
      "Topic 0: checkpoint shelling credit random correctional prisons deaths children detonated disposal constant airstrike swat terminal idaho\n",
      "Topic 1: nord der lithuania tariffs pipeline von baltic iphones tariff germany countries sweden bloc aluminum glasgow\n",
      "Topic 2: race raphael kemp warnock recount ossoff defeats doug electors ins vying jon maloney collins pollsters\n",
      "Topic 3: margaret queue enslaved elegant recounts favorites workings reviews classical accent thriller kennedy essays print reading\n",
      "Topic 4: icu gloves hood mutations item upgraded pneumonia wuhan protein certificates viruses levine matt nursing soap\n",
      "Topic 5: report committee neil clinton special subjects obtain committees nixon mentioned schiff actual phase damaging house\n",
      "Topic 6: overdose evictions overdoses fentanyl debts pills exchanges unpaid homelessness obamacare insurers premiums penalties marketed licensed\n",
      "Topic 7: women children admissions skilled jobs sectors studies disabilities employment mothers workers rural construction students retail\n",
      "Topic 8: herald charleston ferguson abused blacks gunfire lee robinson chased bible laugh anderson policing travis correctional\n",
      "Topic 9: hedge loans partner capitalize managing company creek shrink oversees amazon teamed settlements managers dollars estate\n",
      "Topic 10: party compromise elections wing promote boris parties grace britain parliament pro minister cameron favors parliamentary\n",
      "Topic 11: gas solar offline gases pipelines nord dioxide grid rainfall methodology figuring culturally improvements forests electric\n",
      "Topic 12: estate lots renovation manhattan neighborhood temple penn eat carve houses streets newark food tag tourists\n",
      "Topic 13: playoffs goodell yankees warriors hockey players season athlete griner locker brady finals coaching espn anthem\n",
      "Topic 14: skin twists marco want ineffective obamacare doubles punching honorable grandfather bed hopeful punch honestly machinery\n",
      "Topic 15: gerrymandering discriminatory rights restrictive kemp legislatures incumbents narrower manchin restrict abortion filibuster flipping elections election\n",
      "Topic 16: rajoy mariano decree suspend warsaw separatists champions polish orderly pedro migrants independence retain asylum manchester\n",
      "Topic 17: patricia nuanced recognition prevailing underage dimension lighter recognizes negotiated feels predatory trafficking story agreement sort\n",
      "Topic 18: stereotypes divorced dress loving sympathy oscar dad soul horns doubles films hat excitement affection dream\n",
      "Topic 19: sheldon silver fifa tournaments kickbacks financier bribery disgraced silva testified dealings commission preet subsidiary mit\n",
      "Topic 20: girl walls birth married remembers chapel royal boy forever precious seated remembered story horrific amateur\n",
      "Topic 21: russian influence gone posts twitter borders operates ukrainian videos account campaigns hype allegedly caught indicted\n",
      "Topic 22: withdrawal royal johnson routine matthew unlock duty stepping volunteer queen palace trap union exit regret\n",
      "Topic 23: screening security jihadist isil libya attacker iraqis terrorism iraq airport terminal sudan saddam american militias\n",
      "Topic 24: kashmir barcelona restive graft anniversary rebellion jailed demonstrators carrie fifa yellow ouster unilaterally aleksei oxygen\n",
      "Topic 25: rupert fox anchor defamation resigns slur comedy cable editor journalism rapper rated magazine nbc roger\n",
      "Topic 26: committee house justice devin interfere chairmen officials judiciary nsa investigative nadler subpoena watchdog appointees intelligence\n",
      "Topic 27: finalized wireless arbitration broadband antitrust enrichment sprint deregulation vienna leagues revoke neutrality regulations rules mlb\n",
      "Topic 28: lightfoot assignment rob lori locals mode throat dining slip classmates cafe covid cruise sars respiratory\n",
      "Topic 29: mulvaney giuliani reconstructed inquiry special president attorney bidens lawyer blower zelensky mick mueller counsel trump\n",
      "Topic 30: danish oxygen india surpassed pakistan singapore plunge egypt king pilgrims thailand brazil indian easter globally\n",
      "Topic 31: aids productive fun suggestions learn brand character cook stuff machine digital habit hill realized design\n",
      "Topic 32: queue starring favorites recounts featuring editor commentary essays reviews journalism film enjoying subscriber classical enslaved\n",
      "Topic 33: birds pumping gently plug sustainability hybrid sand cheaper lobby farmer bird gavin cattle store excellent\n",
      "Topic 34: white lagged percent successive southwest utterly latinos sugar clinton notes analyzed shift arguably des lake\n",
      "Topic 35: portland charlie kazakhstan chapters prosecutions slaughter neo agents charred whatsapp xinjiang militia aleksei deployed extremism\n",
      "Topic 36: money joe recording trump bernie archived ben president burns productive graduation hmm mitigate warren unemployment\n",
      "Topic 37: ukraine devin barbaro house impeached erdogan republicans kent hearings ambassador trump meeting michael secretary sondland\n",
      "Topic 38: restive pakistan autonomy bashar assad mosque ruthless factions cleric coup belarus ouster sectarian rabbi syrian\n",
      "Topic 39: peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\n",
      "Topic 40: commission legalize amtrak congestion staffers walt tony mayor blasio sara silver cuomo hamilton assemblyman sheldon\n",
      "Topic 41: confirmation renew yellen marshal chuck nominated signaled secretary chairman speculation powell roy chosen governors casting\n",
      "Topic 42: american assad missiles location russians phase pentagon responsibility united airstrikes weapon ukraine operation weapons vehicle\n",
      "Topic 43: gender children teens students pregnant wear deliberate training student school nurses society teenage babies child\n",
      "Topic 44: church flag instinct practically theory community man crews shoot shot tomorrow just applause crime knock\n",
      "Topic 45: mnuchin panel boehner congress postal nadler recess nsa postmaster senate borrowing obama appeals avert pruitt\n",
      "Topic 46: ethnic setbacks coalition maintained fierce blackout sums vying caps hub funded steered stave departures operates\n",
      "Topic 47: saudi activists rulers arabia clerics prophet ruler anticorruption dissidents kingdom evasion sheikh duchess insults financier\n",
      "Topic 48: percent upshot analytics vote rating respondents survey statistical median surveys cusp exclude rican margin delegate\n",
      "Topic 49: bidens sergey kyiv putin officials gordon rudy nicholas taylor sondland ukrainian zelensky misled informing schmidt\n",
      "Topic 50: siena voters electorate hispanic unfavorable incarcerated hispanics pollsters poll error sample partisanship mrs impression southeast\n",
      "Topic 51: talent howard fox literary rupert comedy intercontinental nbc insider channels celebrity boasting theaters succession valued\n",
      "Topic 52: redacted fbi page subpoenas comey unsealed dept laundering inventory prosecutors surveillance records firing secretly defects\n",
      "Topic 53: stigma unvaccinated herd frustrating ventilation like poor occurring grandchildren kid bit really work guiding care\n",
      "Topic 54: city lease ferry homeless midtown commuters exhibition yorkers dancing acre jewelry plaza art homelessness posters\n",
      "Topic 55: lesser party democratic sugar won income agriculture billionaires cory finishing beto wright switched sanders burns\n",
      "Topic 56: upscale suburban suburbs violence epstein themes cities strongholds disperse uptick reid neighborhood ballot staten culturally\n",
      "Topic 57: gillibrand nbc bros news bannon fox anchor gabriel bullying slate cnn harassment poster kirsten rated\n",
      "Topic 58: wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\n",
      "Topic 59: landslide city letitia offenders accomplishments challenger bharara cuomo mansion primary citywide eric chairs positioned offenses\n",
      "Topic 60: bishop priests cardinal duchess roman church sexually metoo abuse honors dating otto catholic denied priest\n",
      "Topic 61: sinema deceptive loophole cuomo corporations proposition raiser prop contributions tax legalize proposes legalization initiatives campaign\n",
      "Topic 62: data methodology emails registration cybersecurity assessed methods incorrect servers trove carrier adverse databases warrant subpoenas\n",
      "Topic 63: police shells shelling breached qanon frenzied simultaneous animals east mortar battling metro siege insurrection rocket\n",
      "Topic 64: bribes khan silva funnel dictatorship anticorruption province carriers schemes designation leftist rousseff frontier lender overtime\n",
      "Topic 65: injections execution executions injection lethal oklahoma oxycontin trial defendant patent botched drugs opioid samsung corrections\n",
      "Topic 66: mueller special gates accusations allegations markle agent wikileaks email contacts murdoch formerly fake cambridge guilty\n",
      "Topic 67: kidnapping sgt assailants prophet killers kidnapped muhammad hostages nigerian abducted gunned hostage gunmen carnage mosque\n",
      "Topic 68: immigrant stanley ban vital order permanent employees refuse qualified unauthorized workers officials deaths statistics seven\n",
      "Topic 69: publishers hashtag semitic censorship islam ethnicity pioneer goldberg flagged orientation sympathy nyu pundits discourse audiences\n",
      "Topic 70: nuclear iran atomic warmbier uranium tehran korea vessels otto ships north philippines hawks cyberattacks cyber\n",
      "Topic 71: mosul dam kurdish villages river ruins metropolis soar floods mykolaiv miles rainfall acres retake towns\n",
      "Topic 72: biden donation race caucuses poll rubio sanders diminish trailing iowa campaign cruz vermont polling rodham\n",
      "Topic 73: turkish militants monitor claimed fighters chambers tasked shiite controlled attempting backlash gathered measures arrests courthouse\n",
      "Topic 74: rubber protesters supremacists demonstrators slogans protests clashes goodell disperse violent inaction demonstrations lam tear police\n",
      "Topic 75: repercussions recommendations prevented opioid mentally pending weighs agree trumps directive mandate treating barring doctor statement\n",
      "Topic 76: suspension fda afghan ghani ashraf biontech panel shortfall afghanistan coordinator provincial abruptly positive cognitive fed\n",
      "Topic 77: rajoy socialists socialist deadlock pedro inconclusive spain mariano abdullah referendum plunged ghani centrist stalemate impartial\n",
      "Topic 78: pen delegate bend delegates jean des pour caucuses nomination ideologically hat passionate ascent mate pete\n",
      "Topic 79: loretta supported removal remove haley supremacists nationalists clashing gop congresswoman sworn flag kemp lynch takeaways\n",
      "Topic 80: meghan navalny kyiv soldiers zelensky soldier dmitri western blinken contingent ukrainian cold intervention wuhan military\n",
      "Topic 81: said employees work resort anonymous employers members education classmates labor credit learn motivation permanent unions\n",
      "Topic 82: care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\n",
      "Topic 83: gatherings redacted extension johnson downing british servant gray deadline continental mrs liz parliament defy showdown\n",
      "Topic 84: totalitarian openness censors communist echelons reign usher handover china prepares elites chinese jinping ying japan\n",
      "Topic 85: cuomo served graft sullivan politico dept amounted tenure denied banks ashraf holder prosecutor retire abruptly\n",
      "Topic 86: tax manchin filibuster sinema glasgow progressives provisions reconciliation liberals democracies majorities drilling bipartisan bridges ceiling\n",
      "Topic 87: soldiers torture propaganda besieged palestinian eastern border terror proclaimed village occupation getty collective east grim\n",
      "Topic 88: stone meeting jared flynn bannon roger accusations campaign kremlin son paul kushner mrs matthew wikileaks\n",
      "Topic 89: frontier bribes musicians designation blessing vision antigovernment branch pushback tourism nyu insurgency uae restored hezbollah\n",
      "Topic 90: pollsters sampling poll probability polls logic consumer swung positioned gauge alpha earnings increases explanations takeaways\n",
      "Topic 91: water sirens tornadoes residents tornado farms riders busiest generators sidewalks flushing coverings caribbean factories quarantines\n",
      "Topic 92: spells pediatric spark tales cope prevalent depleted toys furniture exceed scarce sustainable goods chunk partial\n",
      "Topic 93: poker lineup strengths receptive button quest swinging netflix smartphone smart justin plug shifts franchise motion\n",
      "Topic 94: okla wis ariz ark ala minn fla ind mich conn del tenn utah colo ore\n",
      "Topic 95: lists connect writer laughed acquiring college wrote tip guided email woman weinstein displayed fun instagram\n",
      "Topic 96: royals poet harry poetry slave sussex acclaimed fictional duchess windsor meghan literature markle novels prose\n",
      "Topic 97: architecture skies singers sung gala opera hardware songs civilization columns pierre paintings productions embraces travels\n",
      "Topic 98: acquisition anthem clubs leagues brands annually format richest jpmorgan rupert super cents mlb premier acquire\n",
      "Topic 99: suleimani flexible benghazi accompanied iraq turf muscular convoy exploding mourning nearing andrew backfire dilemma elderly\n",
      "Topic 100: slowing tesla automaker maker deliveries disruptions markedly quarter constraints samsung motor quarterly curve sales slump\n",
      "Topic 101: wobble rout strings productivity bitcoin acceptance darkness programmers navigating weathering robots marketplace tasks enabling groceries\n",
      "Topic 102: applause bell kyle grace los latino instruction educators descent reservoir chicago latinos congregation asian residents\n",
      "Topic 103: deutsche softbank loan trades pension subsidiary bankers sec loeffler stanley mortgages lenders fargo goldman windfall\n",
      "Topic 104: arabia saudi abbas mattis ghani bahrain reassure arab salman boon boehner qatar persian tillerson pave\n",
      "Topic 105: discourse spends studying historian divorce societies essay nigeria slaughter outfits neglected jonathan mothers halloween inclusion\n",
      "Topic 106: maggie cure archived recording dictate pharma order care postmaster redfield union meaningful safest katie workers\n",
      "Topic 107: vary stream detonated steering meters broadcasting reliance persist monthly reliably terminated thick gas uneven lighter\n",
      "Topic 108: city temple roosevelt pedestrians lease enclave artist wine midtown halloween plaza planners foot subway graffiti\n",
      "Topic 109: constitutionality supreme justices vacancies upheld firearms violates ban regulate unconstitutional injunction cody rulings evictions localities\n",
      "Topic 110: aleksei navalny siberian putin vladimir mykolaiv moscow russia sergei blackout griner nerve poisoning dmitri mourners\n",
      "Topic 111: corroborate members said anonymous satisfied women motivation sources wen education employees learn employers reliable fulfilling\n",
      "Topic 112: extremist terrorist antigovernment guard white counterterrorism capitol missions maj readiness afghanistan paramilitary raids indications shortcomings\n",
      "Topic 113: oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\n",
      "Topic 114: athletes summit dialogue tillerson olympics jae tokyo moon diplomatic winter athens rex pop north pyongyang\n",
      "Topic 115: donation biden tariffs tillerson kenya equitable meeting president pence incited vice blinken supremacists democracies thanked\n",
      "Topic 116: japan base medal consecutive wolf podium episode seventh jae lineup artistic sox olympic broadcast score\n",
      "Topic 117: aleppo isil raqqa kunduz damascus warplanes mykolaiv nigerian mosul haram bombers provincial siberian islamist civilians\n",
      "Topic 118: transgender rebukes gay equality prohibit condemns greene condemn fringe hook discriminatory amendment orientation loretta carolina\n",
      "Topic 119: nepal earthquakes cathedral frantic asleep earthquake stampede tornado nasa tropical tornadoes messenger ripped blaze smashed\n",
      "Topic 120: electors trump hillary mccain mar contests clinton democratic trails masters race gore comfortably christie desantis\n",
      "Topic 121: doctrine court warrant wilson cellphone printed privacy judge affirmative cody sued violating consent unlawful justices\n",
      "Topic 122: cases omicron transmission testing indoor tested caseloads hospitalizations infections quarantines masking gyms transmissible positivity mask\n",
      "Topic 123: ecosystem scott jason sydney wildlife susan vacation southeastern morrison species property bizarre soap trees surreal\n",
      "Topic 124: warmbier missile coma yemeni korea yemen airstrike poisoning otto jong north crown kim korean honors\n",
      "Topic 125: scalise campaign mulvaney candidates ostensibly candidate mccain journalists qanon cover cheney mcconnell supremacists maloney mccarthy\n",
      "Topic 126: citywide york holder cuomo emanuel mayoral upstate adams city symbols mayor ralph assemblyman hochul charleston\n",
      "Topic 127: knife mcdonald gregory manslaughter pistol cameras pinned shootings swinging toronto protester crashes rifles police fatal\n",
      "Topic 128: authoritarian conceal canadians transmissible chinese dynamics import soviet undertaking now batch portraying dynamic mobilize reconcile\n",
      "Topic 129: silva swiss graft evangelical brazilians michel shareholder acquire powerhouse brazil equity fifa stake deutsche alibaba\n",
      "Topic 130: patients medication detect strokes drugs infants cognitive symptoms pill chronic transmit drug muscle consume tests\n",
      "Topic 131: sing writing really comic obsessed lifetime write exciting puzzle belongs beautiful rehearsal storytelling recounts seed\n",
      "Topic 132: unifying meeting elect treats vice oval olympics olympic dear president comments presidency erdogan macron ivanka\n",
      "Topic 133: costumes halloween yale immunized rating pediatrics abuse olds lgbtq couples mit children fetus university profoundly\n",
      "Topic 134: shareholder tabloid hacking transaction publishers bros harassed elon ailes max famously slate bezos murdoch cambridge\n",
      "Topic 135: nominate powell accusations rousseff meddling jerome graft manipulate ashraf cabinet dilma sisi ouster remarks chairwoman\n",
      "Topic 136: merger airline carrier airlines recipients routes ban mentally exempt destinations concentrated carriers exclusively licenses valid\n",
      "Topic 137: insurrection murkowski convict cheney keepers certification stormed wyoming romney boys mitt incited greene ardent proud\n",
      "Topic 138: brexit austerity socialists hollande continent warsaw polish euro rajoy athens merkel humiliating french macedonia membership\n",
      "Topic 139: brad articles vote senate house impeach partisanship election julie president impeached pelosi appointing votes postmaster\n",
      "Topic 140: ambulance certifying gunshots bryan filming whereabouts minutes steal pavement homicide footage reinforcements gunshot wheel keepers\n",
      "Topic 141: veterans speeches vice president jared members campaign kushner values reporters supporting meeting speech barack phrase\n",
      "Topic 142: van entrance switches pacing flies intercept spreads clothes taxi autopsy joking ruthless pulls driveway cairo\n",
      "Topic 143: morale drone soldiers drones occupied atrocities offensive zaporizhzhia volodymyr telegram yulia soldier surrender humanitarian commander\n",
      "Topic 144: motors inland automakers volkswagen balloons electric motor amtrak automaker batteries ford water wheeler autonomous diesel\n",
      "Topic 145: assailants gunmen wounding shootout allegiance compound bernardino stabbed injuring rampage detonated hallway plowed bombings gunshots\n",
      "Topic 146: enriched capability nuclear inspectors terminated atomic tehran collapses tiktok iranian produce treaty huawei ceased inspections\n",
      "Topic 147: intelligence espionage ransomware adversary unspecified anonymous detainee huawei arrested interrogation secrets dissidents pakistani hostage poisoned\n",
      "Topic 148: midterms stampede tornado pilots northeastern siberian nepal evacuation probe cathedral diamond notre bull concert quarantined\n",
      "Topic 149: india indians muslims roman secular fetus liberty religion ultra narendra ethnicity transgender exemption murders religious\n",
      "Topic 150: soaring contraction worshipers shrank deepening britons surpassed economy squeezing gross slump drag australians continent downturn\n",
      "Topic 151: irs moderate poor inflation spells tangible purchasing budgets spell affect tomorrow fared chunk apparel toys\n",
      "Topic 152: pharmacies overdose opioids pharmacy fentanyl municipalities settlement overdoses oxycontin licensed premium addiction drug tribes manufacturer\n",
      "Topic 153: kaiser locked gunman butler christmas rubble closed distancing suddenly outside chaotic heart specialists curtain horses\n",
      "Topic 154: gillibrand harris hampshire kirsten warren mate sanders mayor bend caucuses iowa vermont senator campaign democratic\n",
      "Topic 155: genuine convincing reassuring staring tapes anderson mine proves cheering eyed convinced convince valid raw knees\n",
      "Topic 156: queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\n",
      "Topic 157: snapchat networking mayer verizon kalanick yahoo elon alibaba musk hailing gaming subscription advertisers tiktok cave\n",
      "Topic 158: participants baltimore bleeding trial results trials johnson moderna warp peer incorrect conclusion reviewed company report\n",
      "Topic 159: said reporters members staff white money rallies representative ads stormy causes desantis campaign observers requests\n",
      "Topic 160: manufactured sources distributing india recommendation supplies officials rollout chains raw astrazeneca relying databases freight shortages\n",
      "Topic 161: slide seas totaled screens samsung irma revised dam flooded proclaimed carmaker ford merger inches pitched\n",
      "Topic 162: attorney centered defendant incite felonies guilty lawyers resignations justice charges vacancies acquitted indictment obstruction sheldon\n",
      "Topic 163: extortion subversion assemblyman misdemeanor catholics convicted sheldon towering courthouse indictment financier silver bribery sentenced prominence\n",
      "Topic 164: unanswered royalty bryant actress women flowers abuse oral assaulted metoo rafael jane uncle keith cincinnati\n",
      "Topic 165: holocaust rabbi charlottesville lesbian nationalism neo migrants secular nikki lgbtq haley rally stricter condemns massacre\n",
      "Topic 166: barrels oil atrocities ports mediterranean military ocean persian uae reactor deficits exercises sullivan invading reliant\n",
      "Topic 167: car store fake undercover restaurant asks handgun code videos departments murder photos photo app parking\n",
      "Topic 168: chokehold homicide manslaughter officers mcdonald officer indict minneapolis unarmed derek aggravated offenses misdemeanor handcuffed jurors\n",
      "Topic 169: border countries profitable immigrants migration greed union attractive reductions ross poorer corporations andrew united realization\n",
      "Topic 170: stone ing julian campaign clintons trump publicity informal tricks wikileaks adviser roger writes constitutes insight\n",
      "Topic 171: aviv tel naftali isaac benjamin abbas likud annex diaspora annexation netanyahu inconclusive centrist minister jerusalem\n",
      "Topic 172: facial android extract companies servers data neutrality technology antitrust iphones ransomware computing ibm tiktok click\n",
      "Topic 173: vaccine doses pfizer measles astrazeneca immunity biontech virus vaccination vaccines vaccinated cdc variant dose infection\n",
      "Topic 174: gym uncommon bearing noise photographers atop boom rented avenue colored swimming volunteers tightly roof invaded\n",
      "Topic 175: ashley hesitant violent problematic pneumonia spread solved case spiral breathing situations mail assure violation restraint\n",
      "Topic 176: infuriated commanding endorsement cohn deprived decisive wins impeachment caucuses unify mate primaries alex negotiator delegates\n",
      "Topic 177: broadway acclaimed performer soundtrack poetry phillips musical tony album musician opera singer sang awards song\n",
      "Topic 178: forces retake iraq iraqi captive explosives oct amnesty baghdad monitor base afghans prioritized troops controlled\n",
      "Topic 179: buyers advise modified certificate additional handling intersection discretion dealer housing storage safe effective recovered fearing\n",
      "Topic 180: grocery retailer trade hybrid programmers companies oriented currency globally brands corporate incentive adaptation tariff retailers\n",
      "Topic 181: reopen historically rural nationwide speed college community nowhere california nightmare indiana colleges abolish harvard lgbtq\n",
      "Topic 182: rioters steal mattis pompeo reinforcements generals militia siege rationale cooper maggie impeachment trump qassim launch\n",
      "Topic 183: party ministries ghani win portugal consecutive polling colombia voters won hinder sara parliamentary prime minister\n",
      "Topic 184: nightclub arabic fence hiding pulse freddie exchanged ensued hole gunman von boko singh balls infant\n",
      "Topic 185: schwartz schmidt bell archived story amy blake yelling recording sarah yeah wow cooper julie kate\n",
      "Topic 186: reports timed approximately resolving strongest dates compares missing rolling ore instances calif miss wash informing\n",
      "Topic 187: mutations princess dumped viruses protein dec seattle identical munich cascade collected samples shanghai victoria wuhan\n",
      "Topic 188: abu houthi hezbollah sectarian houthis sana sunni mansour lebanese bahrain insurgency militias qatar yemeni afghans\n",
      "Topic 189: abu superintendent gavin storied residents workers blankets providence collar rents refugee sacramento boko diego mill\n",
      "Topic 190: eligible canada united adult older goal optimism residents mandates hurdle populations pediatric average breakthrough shots\n",
      "Topic 191: unconscious goodman overwhelmed collapses sacrifice storm mob update sooner partially handgun kentucky floor reinforced knock\n",
      "Topic 192: said report credit employees data workers erin supervisors reporting sources york labor businesses expects department\n",
      "Topic 193: hackers confidential disney spying studio nick emails hacker computers cybersecurity communicating traces distributing server disclosing\n",
      "Topic 194: percent yields economic mortgage slowdown outlook stimulus mortgages layoffs bank lending forecasts contraction jobless borrowers\n",
      "Topic 195: appeals circuit abortion court proposition pregnancies convictions unconstitutional rights orleans abortions injunction enforcing kavanaugh justices\n",
      "Topic 196: repairs owed developer collectors devised master fee turnover alternatives preservation inspections leases branches prestigious rewrite\n",
      "Topic 197: retail businesses salary labor tenth workers unions corporate cash loans spending firms work transportation credit\n",
      "Topic 198: gang synagogue scholarship readers enslaved rendered prose slavery massacre essays provocative jesus supremacy whites beatings\n",
      "Topic 199: slam seeded djokovic match cup spectators ball scored matches scoring novak champions finals singles sneakers\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:11<00:00,  5.68s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"L-0_K-0 lonely catholic feminist reign revered editorial politician diaspora equality intellectual newspaper bedrock stirring divorce meghan\": {\n",
      "        \"L-1_K-27 royal queen diana father prince palace actress boy art writer book married ballet harry writers\": [\n",
      "            \"L-2_K-96 royals poet harry poetry slave sussex acclaimed fictional duchess windsor meghan literature markle novels prose\",\n",
      "            \"L-2_K-20 girl walls birth married remembers chapel royal boy forever precious seated remembered story horrific amateur\",\n",
      "            \"L-2_K-177 broadway acclaimed performer soundtrack poetry phillips musical tony album musician opera singer sang awards song\",\n",
      "            \"L-2_K-60 bishop priests cardinal duchess roman church sexually metoo abuse honors dating otto catholic denied priest\"\n",
      "        ],\n",
      "        \"L-1_K-10 chinese china hong kong jinping taiwan beijing communist mainland media mao islands extradition propaganda liu\": [\n",
      "            \"L-2_K-84 totalitarian openness censors communist echelons reign usher handover china prepares elites chinese jinping ying japan\",\n",
      "            \"L-2_K-69 publishers hashtag semitic censorship islam ethnicity pioneer goldberg flagged orientation sympathy nyu pundits discourse audiences\",\n",
      "            \"L-2_K-24 kashmir barcelona restive graft anniversary rebellion jailed demonstrators carrie fifa yellow ouster unilaterally aleksei oxygen\",\n",
      "            \"L-2_K-21 russian influence gone posts twitter borders operates ukrainian videos account campaigns hype allegedly caught indicted\"\n",
      "        ],\n",
      "        \"L-1_K-28 prime minister parliament european britain brexit johnson bloc union exit parliamentary boris netanyahu europe parties\": [\n",
      "            \"L-2_K-10 party compromise elections wing promote boris parties grace britain parliament pro minister cameron favors parliamentary\",\n",
      "            \"L-2_K-138 brexit austerity socialists hollande continent warsaw polish euro rajoy athens merkel humiliating french macedonia membership\",\n",
      "            \"L-2_K-83 gatherings redacted extension johnson downing british servant gray deadline continental mrs liz parliament defy showdown\",\n",
      "            \"L-2_K-171 aviv tel naftali isaac benjamin abbas likud annex diaspora annexation netanyahu inconclusive centrist minister jerusalem\"\n",
      "        ],\n",
      "        \"L-1_K-4 sarah mother jennifer story friends walking man woman daughter guys amazing love guy emotional applause\": [\n",
      "            \"L-2_K-185 schwartz schmidt bell archived story amy blake yelling recording sarah yeah wow cooper julie kate\",\n",
      "            \"L-2_K-18 stereotypes divorced dress loving sympathy oscar dad soul horns doubles films hat excitement affection dream\",\n",
      "            \"L-2_K-28 lightfoot assignment rob lori locals mode throat dining slip classmates cafe covid cruise sars respiratory\",\n",
      "            \"L-2_K-131 sing writing really comic obsessed lifetime write exciting puzzle belongs beautiful rehearsal storytelling recounts seed\"\n",
      "        ],\n",
      "        \"L-1_K-34 hindu confederate jewish jews muslim muslims flag haley rally gay nationalist modi orthodox remove ultra\": [\n",
      "            \"L-2_K-165 holocaust rabbi charlottesville lesbian nationalism neo migrants secular nikki lgbtq haley rally stricter condemns massacre\",\n",
      "            \"L-2_K-149 india indians muslims roman secular fetus liberty religion ultra narendra ethnicity transgender exemption murders religious\",\n",
      "            \"L-2_K-118 transgender rebukes gay equality prohibit condemns greene condemn fringe hook discriminatory amendment orientation loretta carolina\",\n",
      "            \"L-2_K-198 gang synagogue scholarship readers enslaved rendered prose slavery massacre essays provocative jesus supremacy whites beatings\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-1 election vote senate republicans democrats primaries votes senator republican candidates endorsing president mcconnell candidate presidential\": {\n",
      "        \"L-1_K-35 reporters jan rallies house observers president endorse participating capitol causes marches representative rely journalists trump\": [\n",
      "            \"L-2_K-125 scalise campaign mulvaney candidates ostensibly candidate mccain journalists qanon cover cheney mcconnell supremacists maloney mccarthy\",\n",
      "            \"L-2_K-29 mulvaney giuliani reconstructed inquiry special president attorney bidens lawyer blower zelensky mick mueller counsel trump\",\n",
      "            \"L-2_K-137 insurrection murkowski convict cheney keepers certification stormed wyoming romney boys mitt incited greene ardent proud\",\n",
      "            \"L-2_K-159 said reporters members staff white money rallies representative ads stormy causes desantis campaign observers requests\"\n",
      "        ],\n",
      "        \"L-1_K-37 electoral results georgia ballots runoff vote fraud won absentee election generic winner race overturn certified\": [\n",
      "            \"L-2_K-2 race raphael kemp warnock recount ossoff defeats doug electors ins vying jon maloney collins pollsters\",\n",
      "            \"L-2_K-137 insurrection murkowski convict cheney keepers certification stormed wyoming romney boys mitt incited greene ardent proud\",\n",
      "            \"L-2_K-77 rajoy socialists socialist deadlock pedro inconclusive spain mariano abdullah referendum plunged ghani centrist stalemate impartial\",\n",
      "            \"L-2_K-183 party ministries ghani win portugal consecutive polling colombia voters won hinder sara parliamentary prime minister\"\n",
      "        ],\n",
      "        \"L-1_K-30 sanders super voters warren biden bernie primary nominee party cruz polls iowa buttigieg candidacy nomination\": [\n",
      "            \"L-2_K-78 pen delegate bend delegates jean des pour caucuses nomination ideologically hat passionate ascent mate pete\",\n",
      "            \"L-2_K-72 biden donation race caucuses poll rubio sanders diminish trailing iowa campaign cruz vermont polling rodham\",\n",
      "            \"L-2_K-154 gillibrand harris hampshire kirsten warren mate sanders mayor bend caucuses iowa vermont senator campaign democratic\",\n",
      "            \"L-2_K-155 genuine convincing reassuring staring tapes anderson mine proves cheering eyed convinced convince valid raw knees\"\n",
      "        ],\n",
      "        \"L-1_K-26 redistricting legislature abortions ballot abortion supreme rights roe governor voting texas court voter ballots map\": [\n",
      "            \"L-2_K-15 gerrymandering discriminatory rights restrictive kemp legislatures incumbents narrower manchin restrict abortion filibuster flipping elections election\",\n",
      "            \"L-2_K-56 upscale suburban suburbs violence epstein themes cities strongholds disperse uptick reid neighborhood ballot staten culturally\",\n",
      "            \"L-2_K-195 appeals circuit abortion court proposition pregnancies convictions unconstitutional rights orleans abortions injunction enforcing kavanaugh justices\",\n",
      "            \"L-2_K-109 constitutionality supreme justices vacancies upheld firearms violates ban regulate unconstitutional injunction cody rulings evictions localities\"\n",
      "        ],\n",
      "        \"L-1_K-22 bill tax legislation congress cuts income spending taxes budget repeal poor money wealthy senator mnuchin\": [\n",
      "            \"L-2_K-86 tax manchin filibuster sinema glasgow progressives provisions reconciliation liberals democracies majorities drilling bipartisan bridges ceiling\",\n",
      "            \"L-2_K-151 irs moderate poor inflation spells tangible purchasing budgets spell affect tomorrow fared chunk apparel toys\",\n",
      "            \"L-2_K-82 care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\",\n",
      "            \"L-2_K-141 veterans speeches vice president jared members campaign kushner values reporters supporting meeting speech barack phrase\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-2 growth earnings saudi climbed donors index stubbornly rates inflation bank bailey gains jobs yields increases\": {\n",
      "        \"L-1_K-47 subscribers skating toe jumps medal sox netflix quarter ratings triple disney gold cable halfway rep\": [\n",
      "            \"L-2_K-116 japan base medal consecutive wolf podium episode seventh jae lineup artistic sox olympic broadcast score\",\n",
      "            \"L-2_K-161 slide seas totaled screens samsung irma revised dam flooded proclaimed carmaker ford merger inches pitched\",\n",
      "            \"L-2_K-94 okla wis ariz ark ala minn fla ind mich conn del tenn utah colo ore\",\n",
      "            \"L-2_K-39 peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\"\n",
      "        ],\n",
      "        \"L-1_K-46 investors prices stock economy fed markets bond bonds recession market economists rates financial stocks volatility\": [\n",
      "            \"L-2_K-194 percent yields economic mortgage slowdown outlook stimulus mortgages layoffs bank lending forecasts contraction jobless borrowers\",\n",
      "            \"L-2_K-113 oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\",\n",
      "            \"L-2_K-58 wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\",\n",
      "            \"L-2_K-100 slowing tesla automaker maker deliveries disruptions markedly quarter constraints samsung motor quarterly curve sales slump\"\n",
      "        ],\n",
      "        \"L-1_K-16 mohammed kingdom prince bin khashoggi killing crown jamal suspects qaeda hails accusations consulate accused detained\": [\n",
      "            \"L-2_K-47 saudi activists rulers arabia clerics prophet ruler anticorruption dissidents kingdom evasion sheikh duchess insults financier\",\n",
      "            \"L-2_K-67 kidnapping sgt assailants prophet killers kidnapped muhammad hostages nigerian abducted gunned hostage gunmen carnage mosque\",\n",
      "            \"L-2_K-66 mueller special gates accusations allegations markle agent wikileaks email contacts murdoch formerly fake cambridge guilty\",\n",
      "            \"L-2_K-88 stone meeting jared flynn bannon roger accusations campaign kremlin son paul kushner mrs matthew wikileaks\"\n",
      "        ],\n",
      "        \"L-1_K-32 israeli israel shiite insurgents gaza afghan kabul fighters militants airstrikes palestinians palestinian rebel taliban jerusalem\": [\n",
      "            \"L-2_K-188 abu houthi hezbollah sectarian houthis sana sunni mansour lebanese bahrain insurgency militias qatar yemeni afghans\",\n",
      "            \"L-2_K-117 aleppo isil raqqa kunduz damascus warplanes mykolaiv nigerian mosul haram bombers provincial siberian islamist civilians\",\n",
      "            \"L-2_K-171 aviv tel naftali isaac benjamin abbas likud annex diaspora annexation netanyahu inconclusive centrist minister jerusalem\",\n",
      "            \"L-2_K-39 peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\"\n",
      "        ],\n",
      "        \"L-1_K-29 residents school york county angeles fort smith housing community construction jose credit refugees asian mayor\": [\n",
      "            \"L-2_K-189 abu superintendent gavin storied residents workers blankets providence collar rents refugee sacramento boko diego mill\",\n",
      "            \"L-2_K-8 herald charleston ferguson abused blacks gunfire lee robinson chased bible laugh anderson policing travis correctional\",\n",
      "            \"L-2_K-0 checkpoint shelling credit random correctional prisons deaths children detonated disposal constant airstrike swat terminal idaho\",\n",
      "            \"L-2_K-102 applause bell kyle grace los latino instruction educators descent reservoir chicago latinos congregation asian residents\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-3 troops turkey iran kuwait military buildup forces iraqi iranian iraq militias mediterranean zaporizhzhia eastern mykolaiv\": {\n",
      "        \"L-1_K-32 israeli israel shiite insurgents gaza afghan kabul fighters militants airstrikes palestinians palestinian rebel taliban jerusalem\": [\n",
      "            \"L-2_K-188 abu houthi hezbollah sectarian houthis sana sunni mansour lebanese bahrain insurgency militias qatar yemeni afghans\",\n",
      "            \"L-2_K-117 aleppo isil raqqa kunduz damascus warplanes mykolaiv nigerian mosul haram bombers provincial siberian islamist civilians\",\n",
      "            \"L-2_K-171 aviv tel naftali isaac benjamin abbas likud annex diaspora annexation netanyahu inconclusive centrist minister jerusalem\",\n",
      "            \"L-2_K-39 peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\"\n",
      "        ],\n",
      "        \"L-1_K-23 korea missiles north nuclear korean missile kim pentagon iran launch escalate iranians ballistic launched sanctions\": [\n",
      "            \"L-2_K-70 nuclear iran atomic warmbier uranium tehran korea vessels otto ships north philippines hawks cyberattacks cyber\",\n",
      "            \"L-2_K-124 warmbier missile coma yemeni korea yemen airstrike poisoning otto jong north crown kim korean honors\",\n",
      "            \"L-2_K-166 barrels oil atrocities ports mediterranean military ocean persian uae reactor deficits exercises sullivan invading reliant\",\n",
      "            \"L-2_K-182 rioters steal mattis pompeo reinforcements generals militia siege rationale cooper maggie impeachment trump qassim launch\"\n",
      "        ],\n",
      "        \"L-1_K-17 catalonia catalan india delhi oxygen independence barcelona modi migrants mosul rio indian lockdown separatist spain\": [\n",
      "            \"L-2_K-30 danish oxygen india surpassed pakistan singapore plunge egypt king pilgrims thailand brazil indian easter globally\",\n",
      "            \"L-2_K-24 kashmir barcelona restive graft anniversary rebellion jailed demonstrators carrie fifa yellow ouster unilaterally aleksei oxygen\",\n",
      "            \"L-2_K-150 soaring contraction worshipers shrank deepening britons surpassed economy squeezing gross slump drag australians continent downturn\",\n",
      "            \"L-2_K-16 rajoy mariano decree suspend warsaw separatists champions polish orderly pedro migrants independence retain asylum manchester\"\n",
      "        ],\n",
      "        \"L-1_K-36 countries imports exports trade european europe canada russia gas flows natural germany oil britain india\": [\n",
      "            \"L-2_K-1 nord der lithuania tariffs pipeline von baltic iphones tariff germany countries sweden bloc aluminum glasgow\",\n",
      "            \"L-2_K-113 oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\",\n",
      "            \"L-2_K-166 barrels oil atrocities ports mediterranean military ocean persian uae reactor deficits exercises sullivan invading reliant\",\n",
      "            \"L-2_K-169 border countries profitable immigrants migration greed union attractive reductions ross poorer corporations andrew united realization\"\n",
      "        ],\n",
      "        \"L-1_K-1 surfside rodriguez champlain condo rubble survivors rescue searching bodies debris escaped belongings firefighters search towers\": [\n",
      "            \"L-2_K-119 nepal earthquakes cathedral frantic asleep earthquake stampede tornado nasa tropical tornadoes messenger ripped blaze smashed\",\n",
      "            \"L-2_K-153 kaiser locked gunman butler christmas rubble closed distancing suddenly outside chaotic heart specialists curtain horses\",\n",
      "            \"L-2_K-148 midterms stampede tornado pilots northeastern siberian nepal evacuation probe cathedral diamond notre bull concert quarantined\",\n",
      "            \"L-2_K-184 nightclub arabic fence hiding pulse freddie exchanged ensued hole gunman von boko singh balls infant\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-4 now just barbaro know archived michael recording work wow like yeah huh states really want\": {\n",
      "        \"L-1_K-4 sarah mother jennifer story friends walking man woman daughter guys amazing love guy emotional applause\": [\n",
      "            \"L-2_K-185 schwartz schmidt bell archived story amy blake yelling recording sarah yeah wow cooper julie kate\",\n",
      "            \"L-2_K-18 stereotypes divorced dress loving sympathy oscar dad soul horns doubles films hat excitement affection dream\",\n",
      "            \"L-2_K-28 lightfoot assignment rob lori locals mode throat dining slip classmates cafe covid cruise sars respiratory\",\n",
      "            \"L-2_K-131 sing writing really comic obsessed lifetime write exciting puzzle belongs beautiful rehearsal storytelling recounts seed\"\n",
      "        ],\n",
      "        \"L-1_K-43 american violence east border united joe burns andrew won economy climate house streets york suleimani\": [\n",
      "            \"L-2_K-99 suleimani flexible benghazi accompanied iraq turf muscular convoy exploding mourning nearing andrew backfire dilemma elderly\",\n",
      "            \"L-2_K-136 merger airline carrier airlines recipients routes ban mentally exempt destinations concentrated carriers exclusively licenses valid\",\n",
      "            \"L-2_K-55 lesser party democratic sugar won income agriculture billionaires cory finishing beto wright switched sanders burns\",\n",
      "            \"L-2_K-169 border countries profitable immigrants migration greed union attractive reductions ross poorer corporations andrew united realization\"\n",
      "        ],\n",
      "        \"L-1_K-6 weinstein car hill floyd restaurant photos ton fake code hotel annie bystander filming committing arrest\": [\n",
      "            \"L-2_K-167 car store fake undercover restaurant asks handgun code videos departments murder photos photo app parking\",\n",
      "            \"L-2_K-95 lists connect writer laughed acquiring college wrote tip guided email woman weinstein displayed fun instagram\",\n",
      "            \"L-2_K-179 buyers advise modified certificate additional handling intersection discretion dealer housing storage safe effective recovered fearing\",\n",
      "            \"L-2_K-140 ambulance certifying gunshots bryan filming whereabouts minutes steal pavement homicide footage reinforcements gunshot wheel keepers\"\n",
      "        ],\n",
      "        \"L-1_K-12 data rate unemployment voters older poor polls adults higher poll eligible hispanic income voter badly\": [\n",
      "            \"L-2_K-92 spells pediatric spark tales cope prevalent depleted toys furniture exceed scarce sustainable goods chunk partial\",\n",
      "            \"L-2_K-190 eligible canada united adult older goal optimism residents mandates hurdle populations pediatric average breakthrough shots\",\n",
      "            \"L-2_K-50 siena voters electorate hispanic unfavorable incarcerated hispanics pollsters poll error sample partisanship mrs impression southeast\",\n",
      "            \"L-2_K-186 reports timed approximately resolving strongest dates compares missing rolling ore instances calif miss wash informing\"\n",
      "        ],\n",
      "        \"L-1_K-0 committee bill rules call secretary order report conduct constitutional case ambassador security hearings pro witnesses\": [\n",
      "            \"L-2_K-5 report committee neil clinton special subjects obtain committees nixon mentioned schiff actual phase damaging house\",\n",
      "            \"L-2_K-139 brad articles vote senate house impeach partisanship election julie president impeached pelosi appointing votes postmaster\",\n",
      "            \"L-2_K-37 ukraine devin barbaro house impeached erdogan republicans kent hearings ambassador trump meeting michael secretary sondland\",\n",
      "            \"L-2_K-17 patricia nuanced recognition prevailing underage dimension lighter recognizes negotiated feels predatory trafficking story agreement sort\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-5 scene punch festival red bull playing ball yards plate season seconds tall screen camera hip\": {\n",
      "        \"L-1_K-18 players games nfl tournament basketball league championship season fans tennis nba coach player stars baseball\": [\n",
      "            \"L-2_K-13 playoffs goodell yankees warriors hockey players season athlete griner locker brady finals coaching espn anthem\",\n",
      "            \"L-2_K-199 slam seeded djokovic match cup spectators ball scored matches scoring novak champions finals singles sneakers\",\n",
      "            \"L-2_K-116 japan base medal consecutive wolf podium episode seventh jae lineup artistic sox olympic broadcast score\",\n",
      "            \"L-2_K-98 acquisition anthem clubs leagues brands annually format richest jpmorgan rupert super cents mlb premier acquire\"\n",
      "        ],\n",
      "        \"L-1_K-19 film documentary reading magic watching things motives listening movies headlines abstract independents theater movie reviews\": [\n",
      "            \"L-2_K-3 margaret queue enslaved elegant recounts favorites workings reviews classical accent thriller kennedy essays print reading\",\n",
      "            \"L-2_K-32 queue starring favorites recounts featuring editor commentary essays reviews journalism film enjoying subscriber classical enslaved\",\n",
      "            \"L-2_K-156 queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\",\n",
      "            \"L-2_K-177 broadway acclaimed performer soundtrack poetry phillips musical tony album musician opera singer sang awards song\"\n",
      "        ],\n",
      "        \"L-1_K-1 surfside rodriguez champlain condo rubble survivors rescue searching bodies debris escaped belongings firefighters search towers\": [\n",
      "            \"L-2_K-119 nepal earthquakes cathedral frantic asleep earthquake stampede tornado nasa tropical tornadoes messenger ripped blaze smashed\",\n",
      "            \"L-2_K-153 kaiser locked gunman butler christmas rubble closed distancing suddenly outside chaotic heart specialists curtain horses\",\n",
      "            \"L-2_K-148 midterms stampede tornado pilots northeastern siberian nepal evacuation probe cathedral diamond notre bull concert quarantined\",\n",
      "            \"L-2_K-184 nightclub arabic fence hiding pulse freddie exchanged ensued hole gunman von boko singh balls infant\"\n",
      "        ],\n",
      "        \"L-1_K-6 weinstein car hill floyd restaurant photos ton fake code hotel annie bystander filming committing arrest\": [\n",
      "            \"L-2_K-167 car store fake undercover restaurant asks handgun code videos departments murder photos photo app parking\",\n",
      "            \"L-2_K-95 lists connect writer laughed acquiring college wrote tip guided email woman weinstein displayed fun instagram\",\n",
      "            \"L-2_K-179 buyers advise modified certificate additional handling intersection discretion dealer housing storage safe effective recovered fearing\",\n",
      "            \"L-2_K-140 ambulance certifying gunshots bryan filming whereabouts minutes steal pavement homicide footage reinforcements gunshot wheel keepers\"\n",
      "        ],\n",
      "        \"L-1_K-47 subscribers skating toe jumps medal sox netflix quarter ratings triple disney gold cable halfway rep\": [\n",
      "            \"L-2_K-116 japan base medal consecutive wolf podium episode seventh jae lineup artistic sox olympic broadcast score\",\n",
      "            \"L-2_K-161 slide seas totaled screens samsung irma revised dam flooded proclaimed carmaker ford merger inches pitched\",\n",
      "            \"L-2_K-94 okla wis ariz ark ala minn fla ind mich conn del tenn utah colo ore\",\n",
      "            \"L-2_K-39 peninsula tossed lighting korean balcony slammed dam grandson rocket miles stampede aerial spectacular entered wounding\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-6 patients disease covid cdc prevention transmissible symptoms coronavirus dose positive infected intensive virus hospitalized diagnosis\": {\n",
      "        \"L-1_K-38 antibodies immune virologist hepatitis virus treatments drug cells diseases genetic cancer experimental treatment scientists diabetes\": [\n",
      "            \"L-2_K-130 patients medication detect strokes drugs infants cognitive symptoms pill chronic transmit drug muscle consume tests\",\n",
      "            \"L-2_K-173 vaccine doses pfizer measles astrazeneca immunity biontech virus vaccination vaccines vaccinated cdc variant dose infection\",\n",
      "            \"L-2_K-156 queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\",\n",
      "            \"L-2_K-133 costumes halloween yale immunized rating pediatrics abuse olds lgbtq couples mit children fetus university profoundly\"\n",
      "        ],\n",
      "        \"L-1_K-5 inoculated variants ebola epidemiologists variant clinical hamas syndrome vaccinating outweighed newsom boosters contagious booster prescribed\": [\n",
      "            \"L-2_K-173 vaccine doses pfizer measles astrazeneca immunity biontech virus vaccination vaccines vaccinated cdc variant dose infection\",\n",
      "            \"L-2_K-156 queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\",\n",
      "            \"L-2_K-32 queue starring favorites recounts featuring editor commentary essays reviews journalism film enjoying subscriber classical enslaved\",\n",
      "            \"L-2_K-54 city lease ferry homeless midtown commuters exhibition yorkers dancing acre jewelry plaza art homelessness posters\"\n",
      "        ],\n",
      "        \"L-1_K-21 cases spread outbreaks deaths restrictions wuhan texas hospitals quarantine protocols mask masks fined tracing health\": [\n",
      "            \"L-2_K-122 cases omicron transmission testing indoor tested caseloads hospitalizations infections quarantines masking gyms transmissible positivity mask\",\n",
      "            \"L-2_K-187 mutations princess dumped viruses protein dec seattle identical munich cascade collected samples shanghai victoria wuhan\",\n",
      "            \"L-2_K-28 lightfoot assignment rob lori locals mode throat dining slip classmates cafe covid cruise sars respiratory\",\n",
      "            \"L-2_K-4 icu gloves hood mutations item upgraded pneumonia wuhan protein certificates viruses levine matt nursing soap\"\n",
      "        ],\n",
      "        \"L-1_K-17 catalonia catalan india delhi oxygen independence barcelona modi migrants mosul rio indian lockdown separatist spain\": [\n",
      "            \"L-2_K-30 danish oxygen india surpassed pakistan singapore plunge egypt king pilgrims thailand brazil indian easter globally\",\n",
      "            \"L-2_K-24 kashmir barcelona restive graft anniversary rebellion jailed demonstrators carrie fifa yellow ouster unilaterally aleksei oxygen\",\n",
      "            \"L-2_K-150 soaring contraction worshipers shrank deepening britons surpassed economy squeezing gross slump drag australians continent downturn\",\n",
      "            \"L-2_K-16 rajoy mariano decree suspend warsaw separatists champions polish orderly pedro migrants independence retain asylum manchester\"\n",
      "        ],\n",
      "        \"L-1_K-31 insurance rent moratorium eviction tenants medicaid affordable mandate states opioid care renters health rental landlords\": [\n",
      "            \"L-2_K-6 overdose evictions overdoses fentanyl debts pills exchanges unpaid homelessness obamacare insurers premiums penalties marketed licensed\",\n",
      "            \"L-2_K-75 repercussions recommendations prevented opioid mentally pending weighs agree trumps directive mandate treating barring doctor statement\",\n",
      "            \"L-2_K-82 care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\",\n",
      "            \"L-2_K-152 pharmacies overdose opioids pharmacy fentanyl municipalities settlement overdoses oxycontin licensed premium addiction drug tribes manufacturer\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-7 superintendent louisville probation minneapolis unarmed excessive ferguson firearms perpetrators concealed bullets gun incarcerated staten police\": {\n",
      "        \"L-1_K-41 protesters protests videos gunman tear officer arrested protest riot capitol shot shooting video violent violence\": [\n",
      "            \"L-2_K-127 knife mcdonald gregory manslaughter pistol cameras pinned shootings swinging toronto protester crashes rifles police fatal\",\n",
      "            \"L-2_K-74 rubber protesters supremacists demonstrators slogans protests clashes goodell disperse violent inaction demonstrations lam tear police\",\n",
      "            \"L-2_K-145 assailants gunmen wounding shootout allegiance compound bernardino stabbed injuring rampage detonated hallway plowed bombings gunshots\",\n",
      "            \"L-2_K-168 chokehold homicide manslaughter officers mcdonald officer indict minneapolis unarmed derek aggravated offenses misdemeanor handcuffed jurors\"\n",
      "        ],\n",
      "        \"L-1_K-31 insurance rent moratorium eviction tenants medicaid affordable mandate states opioid care renters health rental landlords\": [\n",
      "            \"L-2_K-6 overdose evictions overdoses fentanyl debts pills exchanges unpaid homelessness obamacare insurers premiums penalties marketed licensed\",\n",
      "            \"L-2_K-75 repercussions recommendations prevented opioid mentally pending weighs agree trumps directive mandate treating barring doctor statement\",\n",
      "            \"L-2_K-82 care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\",\n",
      "            \"L-2_K-152 pharmacies overdose opioids pharmacy fentanyl municipalities settlement overdoses oxycontin licensed premium addiction drug tribes manufacturer\"\n",
      "        ],\n",
      "        \"L-1_K-26 redistricting legislature abortions ballot abortion supreme rights roe governor voting texas court voter ballots map\": [\n",
      "            \"L-2_K-15 gerrymandering discriminatory rights restrictive kemp legislatures incumbents narrower manchin restrict abortion filibuster flipping elections election\",\n",
      "            \"L-2_K-56 upscale suburban suburbs violence epstein themes cities strongholds disperse uptick reid neighborhood ballot staten culturally\",\n",
      "            \"L-2_K-195 appeals circuit abortion court proposition pregnancies convictions unconstitutional rights orleans abortions injunction enforcing kavanaugh justices\",\n",
      "            \"L-2_K-109 constitutionality supreme justices vacancies upheld firearms violates ban regulate unconstitutional injunction cody rulings evictions localities\"\n",
      "        ],\n",
      "        \"L-1_K-7 judge case court trial jury sexual lawyers epstein courtroom guilty charges victims prison jail sentence\": [\n",
      "            \"L-2_K-65 injections execution executions injection lethal oklahoma oxycontin trial defendant patent botched drugs opioid samsung corrections\",\n",
      "            \"L-2_K-162 attorney centered defendant incite felonies guilty lawyers resignations justice charges vacancies acquitted indictment obstruction sheldon\",\n",
      "            \"L-2_K-168 chokehold homicide manslaughter officers mcdonald officer indict minneapolis unarmed derek aggravated offenses misdemeanor handcuffed jurors\",\n",
      "            \"L-2_K-163 extortion subversion assemblyman misdemeanor catholics convicted sheldon towering courthouse indictment financier silver bribery sentenced prominence\"\n",
      "        ],\n",
      "        \"L-1_K-29 residents school york county angeles fort smith housing community construction jose credit refugees asian mayor\": [\n",
      "            \"L-2_K-189 abu superintendent gavin storied residents workers blankets providence collar rents refugee sacramento boko diego mill\",\n",
      "            \"L-2_K-8 herald charleston ferguson abused blacks gunfire lee robinson chased bible laugh anderson policing travis correctional\",\n",
      "            \"L-2_K-0 checkpoint shelling credit random correctional prisons deaths children detonated disposal constant airstrike swat terminal idaho\",\n",
      "            \"L-2_K-102 applause bell kyle grace los latino instruction educators descent reservoir chicago latinos congregation asian residents\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-8 billion consumers companies leases debt regulated profits cap subsidy annually partnerships trillion regulatory deposit pricing\": {\n",
      "        \"L-1_K-15 app tech google apple facebook users silicon zuckerberg apps user musk game ads twitter uber\": [\n",
      "            \"L-2_K-157 snapchat networking mayer verizon kalanick yahoo elon alibaba musk hailing gaming subscription advertisers tiktok cave\",\n",
      "            \"L-2_K-172 facial android extract companies servers data neutrality technology antitrust iphones ransomware computing ibm tiktok click\",\n",
      "            \"L-2_K-58 wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\",\n",
      "            \"L-2_K-31 aids productive fun suggestions learn brand character cook stuff machine digital habit hill realized design\"\n",
      "        ],\n",
      "        \"L-1_K-46 investors prices stock economy fed markets bond bonds recession market economists rates financial stocks volatility\": [\n",
      "            \"L-2_K-194 percent yields economic mortgage slowdown outlook stimulus mortgages layoffs bank lending forecasts contraction jobless borrowers\",\n",
      "            \"L-2_K-113 oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\",\n",
      "            \"L-2_K-58 wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\",\n",
      "            \"L-2_K-100 slowing tesla automaker maker deliveries disruptions markedly quarter constraints samsung motor quarterly curve sales slump\"\n",
      "        ],\n",
      "        \"L-1_K-36 countries imports exports trade european europe canada russia gas flows natural germany oil britain india\": [\n",
      "            \"L-2_K-1 nord der lithuania tariffs pipeline von baltic iphones tariff germany countries sweden bloc aluminum glasgow\",\n",
      "            \"L-2_K-113 oil commodity traders petroleum barrel gdp commodities barrels output inflation percent benchmark slowdown wholesale budgets\",\n",
      "            \"L-2_K-166 barrels oil atrocities ports mediterranean military ocean persian uae reactor deficits exercises sullivan invading reliant\",\n",
      "            \"L-2_K-169 border countries profitable immigrants migration greed union attractive reductions ross poorer corporations andrew united realization\"\n",
      "        ],\n",
      "        \"L-1_K-31 insurance rent moratorium eviction tenants medicaid affordable mandate states opioid care renters health rental landlords\": [\n",
      "            \"L-2_K-6 overdose evictions overdoses fentanyl debts pills exchanges unpaid homelessness obamacare insurers premiums penalties marketed licensed\",\n",
      "            \"L-2_K-75 repercussions recommendations prevented opioid mentally pending weighs agree trumps directive mandate treating barring doctor statement\",\n",
      "            \"L-2_K-82 care obamacare bill premiums repeal trillion exempt caps repealing tax partial reconciliation insurers health cuts\",\n",
      "            \"L-2_K-152 pharmacies overdose opioids pharmacy fentanyl municipalities settlement overdoses oxycontin licensed premium addiction drug tribes manufacturer\"\n",
      "        ],\n",
      "        \"L-1_K-3 panel regulations pruitt rules obama privacy agency emissions rule telecommunications epa environmental regulatory regulation arbitration\": [\n",
      "            \"L-2_K-27 finalized wireless arbitration broadband antitrust enrichment sprint deregulation vienna leagues revoke neutrality regulations rules mlb\",\n",
      "            \"L-2_K-45 mnuchin panel boehner congress postal nadler recess nsa postmaster senate borrowing obama appeals avert pruitt\",\n",
      "            \"L-2_K-146 enriched capability nuclear inspectors terminated atomic tehran collapses tiktok iranian produce treaty huawei ceased inspections\",\n",
      "            \"L-2_K-109 constitutionality supreme justices vacancies upheld firearms violates ban regulate unconstitutional injunction cody rulings evictions localities\"\n",
      "        ]\n",
      "    },\n",
      "    \"L-0_K-9 investigation lawyer complaint investigators counsel inquiry investigations testimony subpoena prosecutors witness documents wrongdoing misled cohen\": {\n",
      "        \"L-1_K-8 counterintelligence manafort page mueller rudolph comey special iii barr subpoenaed recused attorney fixer giuliani interference\": [\n",
      "            \"L-2_K-52 redacted fbi page subpoenas comey unsealed dept laundering inventory prosecutors surveillance records firing secretly defects\",\n",
      "            \"L-2_K-29 mulvaney giuliani reconstructed inquiry special president attorney bidens lawyer blower zelensky mick mueller counsel trump\",\n",
      "            \"L-2_K-66 mueller special gates accusations allegations markle agent wikileaks email contacts murdoch formerly fake cambridge guilty\",\n",
      "            \"L-2_K-156 queue scans pathogen delivers lately interviewing monkeypox slide duchess humor download harry margaret sussex testifying\"\n",
      "        ],\n",
      "        \"L-1_K-7 judge case court trial jury sexual lawyers epstein courtroom guilty charges victims prison jail sentence\": [\n",
      "            \"L-2_K-65 injections execution executions injection lethal oklahoma oxycontin trial defendant patent botched drugs opioid samsung corrections\",\n",
      "            \"L-2_K-162 attorney centered defendant incite felonies guilty lawyers resignations justice charges vacancies acquitted indictment obstruction sheldon\",\n",
      "            \"L-2_K-168 chokehold homicide manslaughter officers mcdonald officer indict minneapolis unarmed derek aggravated offenses misdemeanor handcuffed jurors\",\n",
      "            \"L-2_K-163 extortion subversion assemblyman misdemeanor catholics convicted sheldon towering courthouse indictment financier silver bribery sentenced prominence\"\n",
      "        ],\n",
      "        \"L-1_K-16 mohammed kingdom prince bin khashoggi killing crown jamal suspects qaeda hails accusations consulate accused detained\": [\n",
      "            \"L-2_K-47 saudi activists rulers arabia clerics prophet ruler anticorruption dissidents kingdom evasion sheikh duchess insults financier\",\n",
      "            \"L-2_K-67 kidnapping sgt assailants prophet killers kidnapped muhammad hostages nigerian abducted gunned hostage gunmen carnage mosque\",\n",
      "            \"L-2_K-66 mueller special gates accusations allegations markle agent wikileaks email contacts murdoch formerly fake cambridge guilty\",\n",
      "            \"L-2_K-88 stone meeting jared flynn bannon roger accusations campaign kremlin son paul kushner mrs matthew wikileaks\"\n",
      "        ],\n",
      "        \"L-1_K-15 app tech google apple facebook users silicon zuckerberg apps user musk game ads twitter uber\": [\n",
      "            \"L-2_K-157 snapchat networking mayer verizon kalanick yahoo elon alibaba musk hailing gaming subscription advertisers tiktok cave\",\n",
      "            \"L-2_K-172 facial android extract companies servers data neutrality technology antitrust iphones ransomware computing ibm tiktok click\",\n",
      "            \"L-2_K-58 wework valuation softbank tumble tesla rout weathering snap batteries wobble market upstart bet offerings quarterly\",\n",
      "            \"L-2_K-31 aids productive fun suggestions learn brand character cook stuff machine digital habit hill realized design\"\n",
      "        ],\n",
      "        \"L-1_K-35 reporters jan rallies house observers president endorse participating capitol causes marches representative rely journalists trump\": [\n",
      "            \"L-2_K-125 scalise campaign mulvaney candidates ostensibly candidate mccain journalists qanon cover cheney mcconnell supremacists maloney mccarthy\",\n",
      "            \"L-2_K-29 mulvaney giuliani reconstructed inquiry special president attorney bidens lawyer blower zelensky mick mueller counsel trump\",\n",
      "            \"L-2_K-137 insurrection murkowski convict cheney keepers certification stormed wyoming romney boys mitt incited greene ardent proud\",\n",
      "            \"L-2_K-159 said reporters members staff white money rallies representative ads stormy causes desantis campaign observers requests\"\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{'PCC': -0.03704572925804966, 'PCD': 0.9575, 'Sibling_TD': 0.9663333333333334, 'PnCD': 0.9986636432350718}\n"
     ]
    }
   ],
   "source": [
    "# evaluate quality of topic hierarchy\n",
    "beta_list = trainer.export_beta()\n",
    "phi_list = trainer.export_phi()\n",
    "annoated_top_words = trainer.export_top_words(dataset.vocab, annotation=True)\n",
    "reference_bow = np.concatenate((dataset.train_bow, dataset.test_bow), axis=0) # or reference_bow = train_bow\n",
    "results, topic_hierarchy = evaluations.hierarchy_quality(dataset.vocab, reference_bow, annoated_top_words, beta_list, phi_list)\n",
    "\n",
    "print(json.dumps(topic_hierarchy, indent=4))\n",
    "print(results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch1.13py3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}