tutorials/tutorial_basic_topic_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://raw.githubusercontent.com/BobXWu/TopMost/master/data/20NG.zip\n",
      "Downloading https://raw.githubusercontent.com/BobXWu/TopMost/master/data/20NG.zip to ./datasets/20NG.zip\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11927380/11927380 [00:00<00:00, 17930247.36it/s]\n"
     ]
    }
   ],
   "source": [
    "import topmost\n",
    "from topmost.data import download_dataset\n",
    "\n",
    "device = \"cuda\" # or \"cpu\"\n",
    "dataset_dir = \"./datasets/20NG\"\n",
    "download_dataset('20NG', cache_path='./datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>train_size:  11314\n",
      "===>test_size:  7532\n",
      "===>vocab_size:  5000\n",
      "===>average length: 110.543\n"
     ]
    }
   ],
   "source": [
    "########################### Neural Topic Models ####################################\n",
    "# dataset for neural topic models.\n",
    "# For combinedTM, add contextual_embed=True.\n",
    "dataset = topmost.data.BasicDatasetHandler(dataset_dir, device=device, read_labels=True, as_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 6/200 [00:00<00:31,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 005 loss: 951.823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 11/200 [00:01<00:30,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 010 loss: 886.403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 16/200 [00:02<00:29,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 015 loss: 865.750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 21/200 [00:03<00:28,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 020 loss: 859.112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 26/200 [00:04<00:28,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 025 loss: 855.170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 31/200 [00:05<00:27,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 030 loss: 852.145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 36/200 [00:05<00:26,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 035 loss: 849.681\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 41/200 [00:06<00:19,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 040 loss: 847.823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 46/200 [00:07<00:17,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 045 loss: 846.598\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 51/200 [00:07<00:16,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 050 loss: 845.313\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 56/200 [00:08<00:16,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 055 loss: 843.899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 61/200 [00:08<00:15,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 060 loss: 842.962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 66/200 [00:09<00:14,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 065 loss: 842.002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 71/200 [00:09<00:14,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 070 loss: 841.132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 76/200 [00:10<00:13,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 075 loss: 840.411\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 81/200 [00:10<00:13,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 080 loss: 839.715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 86/200 [00:11<00:12,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 085 loss: 838.985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 91/200 [00:12<00:12,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 090 loss: 838.475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 96/200 [00:12<00:11,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 095 loss: 837.727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 101/200 [00:13<00:11,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 100 loss: 837.161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 106/200 [00:13<00:10,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 105 loss: 836.693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 111/200 [00:14<00:10,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 110 loss: 836.197\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 116/200 [00:14<00:09,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 115 loss: 835.578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 121/200 [00:15<00:08,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 120 loss: 835.341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 126/200 [00:15<00:08,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 125 loss: 834.705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 131/200 [00:16<00:07,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 130 loss: 834.286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 136/200 [00:17<00:07,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 135 loss: 833.941\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 141/200 [00:17<00:06,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 140 loss: 833.485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 146/200 [00:18<00:06,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 145 loss: 832.925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 151/200 [00:18<00:05,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 150 loss: 832.595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 156/200 [00:19<00:04,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 155 loss: 832.146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 161/200 [00:19<00:04,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 160 loss: 831.726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 166/200 [00:20<00:03,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 165 loss: 831.283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 171/200 [00:21<00:03,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 170 loss: 830.910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 176/200 [00:21<00:02,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 175 loss: 830.437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 181/200 [00:22<00:02,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 180 loss: 830.088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 186/200 [00:22<00:01,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 185 loss: 829.739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 191/200 [00:23<00:01,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 190 loss: 829.443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 196/200 [00:23<00:00,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 195 loss: 829.174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:24<00:00,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 200 loss: 828.991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# create a model\n",
    "# model = topmost.models.ProdLDA(dataset.vocab_size)\n",
    "model = topmost.models.ETM(dataset.vocab_size, pretrained_WE=dataset.pretrained_WE)\n",
    "# model = topmost.models.DecTM(dataset.vocab_size)\n",
    "# model = topmost.models.TSCTM(dataset.vocab_size)\n",
    "# model = topmost.models.CombinedTM(dataset.vocab_size, dataset.contextual_embed_size)\n",
    "# model = topmost.models.NSTM(dataset.vocab_size, pretrained_WE=dataset.pretrained_WE)\n",
    "# model = topmost.models.ECRTM(dataset.vocab_size, pretrained_WE=dataset.pretrained_WE)\n",
    "model = model.to(device)\n",
    "\n",
    "# create a trainer\n",
    "trainer = topmost.trainers.BasicTrainer(model)\n",
    "\n",
    "# train the model\n",
    "trainer.train(dataset)\n",
    "\n",
    "# Or directly use fit_transform\n",
    "# top_words, train_theta = trainer.fit_transform(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0: problem problems cause correct getting water else bad unless anyway causes patients oil otherwise due\n",
      "Topic 1: card scsi mac memory machine ibm writes controller ram modem ide appreciated cards printer port\n",
      "Topic 2: question read writes note world opinions true questions send answer eric issue brian comments wrote\n",
      "Topic 3: subject anyone believe wanted ever someone perhaps mind seem agree probably possibly suspect seems certainly\n",
      "Topic 4: dos pub univ originator serdar batf lib uucp vax vms btw duo isa quadra vlb\n",
      "Topic 5: drive bit gun hard disk control apple drives guns sort weapons rate least firearms self\n",
      "Topic 6: key encryption law public chip security government federal rights keys use privacy access secret enforcement\n",
      "Topic 7: nntp posting previous germany pro jewish jump custom era norway breaking gods past stone helsinki\n",
      "Topic 8: new since years news old times april last numbers york recently ago manager show article\n",
      "Topic 9: know john anyone michael robert dave paul mike james tom fbi peter anybody ken thomas\n",
      "Topic 10: also one many case called part made example however based several fact president support current\n",
      "Topic 11: time post run sun order open possible make without top date return check start andrew\n",
      "Topic 12: host university institute newsreader cambridge watch miss show watching professor polytechnic netherlands graduate enjoy live\n",
      "Topic 13: organization inc clipper group corporation serial motorola purdue sci tektronix technologies forever based cult erik\n",
      "Topic 14: etc information mail available server ftp faq network list internet anonymous stuff email mode computing\n",
      "Topic 15: like unix fax contact interested help dan books shipping including related manual math various tools\n",
      "Topic 16: people work just much better get actually make least many cost writes one works done\n",
      "Topic 17: dept earth black light moon goes lunar surface blue object effect solar tel hole draw\n",
      "Topic 18: organization israel israeli jews arab hope peace today never state states little jewish palestinian work\n",
      "Topic 19: thanks lines number line two following set distribution large via area small western point error\n",
      "Topic 20: government clinton insurance state business private public house money department course care policy need tax\n",
      "Topic 21: space center non national nasa advance air launch ground high service commercial international station satellite\n",
      "Topic 22: reply university mail please phone drivers driver mine address radio college gmt chris live medical\n",
      "Topic 23: get need want help try call find give apr message end access tried trying health\n",
      "Topic 24: file program files code programs display graphics software motif color create directory version application copy\n",
      "Topic 25: like car speed seen bike bus fast cars makes road engineering think big engine corp\n",
      "Topic 26: system good sure local technology means see sound know board great really still anyone quality\n",
      "Topic 27: distribution usa free canada steve except plus washington lab jason drugs georgia ten none jeff\n",
      "Topic 28: article jim dod atheists sorry nice cover tim scott pat atheism simms think greg meg\n",
      "Topic 29: now see right name look looking keep real find change comes say looks word body\n",
      "Topic 30: just got time long even back put around still get come never far couple much\n",
      "Topic 31: box california texas baseball george uunet disclaimer virginia gateway interested rochester leafs somewhere fine folks\n",
      "Topic 32: game team games play season hockey win good player players league nhl best teams just\n",
      "Topic 33: god jesus believe people one say think life even christ know bible christian true something\n",
      "Topic 34: bill pittsburgh gordon gary banks hall winnipeg rules flames flyers state calgary list reserve total\n",
      "Topic 35: one like way just either best course running hand ones rather even right time another\n",
      "Topic 36: science research information general scientific project discussion des published program knowledge objective studies theory analysis\n",
      "Topic 37: article david keywords please posting info entry posted email author traffic disclaimer quotes fax exit\n",
      "Topic 38: computer windows software window internet net bbs summary mit wire performance application microsoft email screen\n",
      "Topic 39: max power monitor marc monitors helmet control fax controls speaker ama timer programmer satan staff\n",
      "Topic 40: people war turkish armenian armenians one killed jews soviet civilians president turks turkey country many\n",
      "Topic 41: lines send thread connect rich screw tapped threads greetings ship tap cables roughly connections voice\n",
      "Topic 42: know think say yes says good mean buy something someone guess remember really tell maybe\n",
      "Topic 43: first year world last one second next san sale two years american toronto chicago another\n",
      "Topic 44: writes mark bob keith smith frank jon heard doug richard wrote steve ron ryan joseph\n",
      "Topic 45: data systems high system used low test level higher technology built design designed pin power\n",
      "Topic 46: said day people every going days started told man state found department book reading police\n",
      "Topic 47: use used using version standard source video output type support include mouse available original uses\n",
      "Topic 48: image price new info list package model deleted images tin digital jpeg includes rom archive\n",
      "Topic 49: home division lost illinois joe face roger guy point van larry street urbana mon cal\n",
      "TD: 0.74400\n",
      "{'Purity': 0.3271375464684015, 'NMI': 0.30523226092357575}\n",
      "{'acc': 0.492299522039299, 'macro-F1': 0.479449539807395}\n"
     ]
    }
   ],
   "source": [
    "########################### Evaluate ####################################\n",
    "# get theta (doc-topic distributions)\n",
    "train_theta, test_theta = trainer.export_theta(dataset)\n",
    "\n",
    "# get top words of topics\n",
    "top_words = trainer.export_top_words(dataset.vocab)\n",
    "\n",
    "\n",
    "# evaluate topic coherence\n",
    "# refer to https://github.com/BobXWu/ECRTM\n",
    "\n",
    "# evaluate topic diversity\n",
    "TD = topmost.evaluations.compute_topic_diversity(top_words, _type=\"TD\")\n",
    "print(f\"TD: {TD:.5f}\")\n",
    "\n",
    "# evaluate clustering\n",
    "results = topmost.evaluations.evaluate_clustering(test_theta, dataset.test_labels)\n",
    "print(results)\n",
    "\n",
    "# evaluate classification\n",
    "results = topmost.evaluations.evaluate_classification(train_theta, test_theta, dataset.train_labels, dataset.test_labels)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://raw.githubusercontent.com/BobXWu/TopMost/master/data/stopwords.zip\n",
      "Downloading https://raw.githubusercontent.com/BobXWu/TopMost/master/data/stopwords.zip to ./datasets/stopwords.zip\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14490/14490 [00:00<00:00, 13651272.45it/s]\n",
      "===>parse texts: 100%|██████████| 2/2 [00:00<00:00, 2722.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.66042987e-03 7.71789113e-04 8.25646392e-04 1.10463367e-03\n",
      "  4.78153583e-04 3.65008309e-04 1.13710971e-03 5.09790785e-04\n",
      "  2.93059386e-02 3.12569959e-04 8.19714647e-03 2.57630204e-03\n",
      "  6.06952060e-04 8.41475674e-04 7.37926224e-04 1.12655126e-02\n",
      "  1.15848170e-03 2.90795811e-03 1.69705204e-03 1.04231131e-03\n",
      "  1.56196859e-03 8.40799928e-01 4.44931618e-04 7.88813573e-04\n",
      "  1.33758923e-03 7.46921636e-04 9.82839032e-04 1.12283078e-03\n",
      "  5.13548090e-04 3.16631701e-03 2.84089893e-03 6.25760003e-04\n",
      "  8.53457197e-04 1.37554342e-03 5.78266627e-04 5.35720866e-03\n",
      "  5.14276735e-02 3.63191037e-04 6.96261588e-04 1.13682728e-03\n",
      "  1.45172491e-03 7.36083253e-04 7.61771691e-04 3.57836136e-03\n",
      "  7.35451060e-04 3.88829573e-03 1.25158031e-03 1.26353174e-03\n",
      "  8.76985374e-04 1.23327610e-03]\n",
      " [1.40189519e-03 2.34943605e-03 1.87826296e-03 2.04059947e-03\n",
      "  1.37949837e-02 2.07699859e-03 3.54039948e-03 1.10803836e-03\n",
      "  3.03937662e-02 2.04983982e-03 1.69548299e-02 2.26524938e-03\n",
      "  9.80712241e-04 2.09111883e-03 1.71996711e-03 2.48499289e-02\n",
      "  3.76371993e-03 1.68262550e-03 1.07555534e-03 1.24433881e-03\n",
      "  9.31609073e-04 9.24371998e-04 8.37141415e-04 1.88015564e-03\n",
      "  4.93435562e-02 1.31300208e-03 2.23323167e-03 1.36383029e-03\n",
      "  9.06893751e-04 3.40966252e-03 2.12128600e-03 1.05168915e-03\n",
      "  1.83423364e-03 2.84454133e-03 1.95146666e-03 9.11391620e-03\n",
      "  1.39880907e-02 1.14722061e-03 7.61996686e-01 1.61731453e-03\n",
      "  3.30890808e-03 1.44976308e-03 4.08870215e-03 1.65983126e-03\n",
      "  1.24306791e-03 4.42981487e-03 1.18614989e-03 9.90681234e-04\n",
      "  1.43165514e-03 2.13937345e-03]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "########################### test new documents ####################################\n",
    "import torch\n",
    "from topmost.preprocessing import Preprocessing\n",
    "\n",
    "preprocessing = Preprocessing()\n",
    "# download_dataset('stopwords', cache_path='./datasets')\n",
    "# preprocessing = Preprocessing(vocab_size=5000, stopwords='./datasets/stopwords/snowball_stopwords.txt')\n",
    "\n",
    "new_docs = [\n",
    "    \"This is a new document about space, including words like space, satellite, launch, orbit.\",\n",
    "    \"This is a new document about Microsoft Windows, including words like windows, files, dos.\"\n",
    "]\n",
    "\n",
    "parsed_new_docs, new_bow = preprocessing.parse(new_docs, vocab=dataset.vocab)\n",
    "new_doc_topic_dist = trainer.test(torch.as_tensor(new_bow, device=device).float())\n",
    "\n",
    "print(new_doc_topic_dist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch1.13py3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}