hansard_topics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing helper functions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.utils:loading LdaModel object from /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load lda model from /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.utils:loading expElogbeta from /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model.expElogbeta.npy with mmap=None\n",
      "INFO:gensim.utils:setting ignored attribute dispatcher to None\n",
      "INFO:gensim.utils:setting ignored attribute state to None\n",
      "INFO:gensim.utils:setting ignored attribute id2word to None\n",
      "INFO:gensim.utils:loaded /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model\n",
      "INFO:gensim.utils:loading LdaState object from /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model.state\n",
      "INFO:gensim.utils:loaded /home/azureuser/cloudfiles/code/data/processing/hansard/experiment/eval_06/lda.model.state\n"
     ]
    }
   ],
   "source": [
    "# %%\n",
    "import helpers.process as process\n",
    "from helpers import io as pickle_io\n",
    "from gensim.models import ldamodel\n",
    "from gensim.models import CoherenceModel\n",
    "import pyLDAvis\n",
    "import tqdm\n",
    "from helpers import hansard\n",
    "import gc\n",
    "import logging\n",
    "import numpy as np\n",
    "logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)\n",
    "\n",
    "\n",
    "root_folder = hansard.rootFolder('eval_06')\n",
    "\n",
    "lda_model_fn = root_folder + '/lda.model'\n",
    "print(f'load lda model from {lda_model_fn}')\n",
    "lda_model = ldamodel.LdaModel.load(lda_model_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%\n",
    "#  print topics and top terms in topic\n",
    "def print_topic(lda_model, topicid, n):\n",
    "    topic_terms = lda_model.get_topic_terms(topicid, topn=n)\n",
    "    terms = list()\n",
    "    probs = list()\n",
    "    for termid, prob in topic_terms:\n",
    "        term = lda_model.id2word.get(termid)\n",
    "        terms.append(term)\n",
    "        probs.append(prob)\n",
    "    terms = ' '.join(terms)\n",
    "    print(f'Topic {topicid:02}: {terms}')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 00: child need support disability family parent work care life service\n",
      "Topic 01: committee recommendation inquiry report review recommendation_committee including committee_recommends program recommends\n",
      "Topic 02: child northern_territory aboriginal_torres strait_islander indigenous community aboriginal abuse territory first_nation\n",
      "Topic 03: community council fire know local need bank home disaster support\n",
      "Topic 04: student university higher_education education course support provider sector system need\n",
      "Topic 05: fuel need colbeck industry community know like change work going\n",
      "Topic 06: great year know like first world life history country made\n",
      "Topic 07: industry ship shipping flag port live_export shipping_industry trade seafarer export\n",
      "Topic 08: reef great_barrier water area vessel environment park fishery specie management\n",
      "Topic 09: conroy mckenzie leader_national sunset_clause clark pyne box black_economy parry office\n",
      "Topic 10: economy business job labor worker country cent know small_business australian\n",
      "Topic 11: morrison third agreed_read read_third levy honourable_follows read_first total proceed_without expenditure\n",
      "Topic 12: nash commonwealth japan relation important preventative_health interest country need system\n",
      "Topic 13: tourism industry know country tourism_industry need support backpacker like want\n",
      "Topic 14: liberal queensland donation political natale party political_donation political_party green public\n",
      "Topic 15: budget labor know cent going cut australian need year country\n",
      "Topic 16: telstra service network broadband rollout telecommunication consumer technology access medicine\n",
      "Topic 17: citizenship visa migration border country border_protection island person immigration community\n",
      "Topic 18: election sport vote voter electoral electoral_commission roll passport voting state\n",
      "Topic 19: committee report inquiry member agreed standing_order recommendation order relation review\n",
      "Topic 20: pension deputy_order plastic pensioner recycling leave_granted excise commend_adjourned waste product_stewardship\n",
      "Topic 21: tasmania state tasmanian hospital tasmanians service health funding need federal\n",
      "Topic 22: school education funding student child need labor teacher know state\n",
      "Topic 23: occupational_health claim work area worker_compensation safety state system know going\n",
      "Topic 24: community year support life australia drug family electorate child service\n",
      "Topic 25: prime queensland gold_coast tobacco smoking cigarette governor_general confirm council given\n",
      "Topic 26: road infrastructure south_wale area state need city project funding electorate\n",
      "Topic 27: industry trade country investment business agreement support opportunity economy export\n",
      "Topic 28: payment support measure family social_security income_support young trial youth_allowance card\n",
      "Topic 29: medium broadcaster service content advertising television broadcasting broadcast digital radio\n",
      "Topic 30: woman work need training employment program domestic_violence employer support job\n",
      "Topic 31: area electorate town community going country queensland know rural_regional fact\n",
      "Topic 32: family child child_care parent early_childhood mother care baby support need\n",
      "Topic 33: worker work fair_work labor employer wage system working employee know\n",
      "Topic 34: court commission ombudsman system work commonwealth employee family_court support reform\n",
      "Topic 35: aged_care defence service care facility need defence_force support home_care older_australian\n",
      "Topic 36: water murray_darling agreement basin fifield river need system free_trade south\n",
      "Topic 37: regional western_sydney region community program need sydney regional_community future_fund rural_regional\n",
      "Topic 38: gonski area community indigenous_student need important support work program film\n",
      "Topic 39: leader south policy industry know labor job country cent come\n",
      "Topic 40: community funding program need area support local project fund work\n",
      "Topic 41: mental_health service health support community program including work centre treatment\n",
      "Topic 42: fact know case report inquiry department could relation particular response\n",
      "Topic 43: fund company superannuation business income investment change member superannuation_fund asset\n",
      "Topic 44: medium labor broadband news journalist need journalism service price network\n",
      "Topic 45: pandemic disease biosecurity vaccine health virus custom employee work outbreak\n",
      "Topic 46: service cost contract department office provided staff financial number post\n",
      "Topic 47: point_order order answer brandis question asked resume_seat call chamber going\n",
      "Topic 48: leave scheme paid_parental entitlement leave_scheme week parental_leave work family employer\n",
      "Topic 49: zealand country refugee program policy cook come need fact going\n",
      "Topic 50: family child_care child family_benefit payment system benefit debt income centrelink\n",
      "Topic 51: bank asic financial_service regulator commission royal_commission consumer corporation regulation provision\n",
      "Topic 52: including project system cost development report industry program work research\n",
      "Topic 53: security country abortion bali need australian work important know area\n",
      "Topic 54: abbott know policy member_interjecting prime want member_left isaac leader member_opposite\n",
      "Topic 55: birmingham efic policy know prime iraq rudd interest going saddam_hussein\n",
      "Topic 56: project western state infrastructure going federal labor need investment commonwealth\n",
      "Topic 57: human_right indigenous bilyk private_session right discrimination indigenous_community community support commission\n",
      "Topic 58: food board director grower authority tehan chemical company wheat standard\n",
      "Topic 59: site land area public_work building working_holiday work development facility property\n",
      "Topic 60: answer home_affair relation advice asked temporary_chair carr parliamentary_secretary department milne\n",
      "Topic 61: energy industry renewable_energy clean_energy target need cost business policy future\n",
      "Topic 62: labor prime know want going today leader like election fact\n",
      "Topic 63: native_title land heavy_vehicle indigenous_land tribunal charge traditional_owner road_transport holder lease\n",
      "Topic 64: know land cattle iraq prime_agricultural cycling going country year want\n",
      "Topic 65: service payment centre community program support family funding subsidy artist\n",
      "Topic 66: page_line schedule_item airport substitute omit aircraft offence flight sydney_airport item_page\n",
      "Topic 67: union worker business labor company work trade_union employer royal_commission know\n",
      "Topic 68: small_business business joyce cancer support breast_cancer know small owner tape\n",
      "Topic 69: asylum_seeker nauru refugee country policy know detention come fact boat\n",
      "Topic 70: person section page_line subsection relation part schedule_item substitute insert clause_page\n",
      "Topic 71: business region china area great chinese well need country like\n",
      "Topic 72: norfolk_island asked_representing centre department_agency upon_notice child_care electoral_division department_home financial service\n",
      "Topic 73: climate_change change emission emission_trading scheme climate country system need world\n",
      "Topic 74: community know life family need like australian work today want\n",
      "Topic 75: sherry lundy longer_heard white_ribbon fund superannuation contribution surcharge heard labor\n",
      "Topic 76: rudd_labor student school family need rhiannon education place system support\n",
      "Topic 77: regulation amendment concern change process relation need review support important\n",
      "Topic 78: department whish_wilson organisation grant application siewert program funding commonwealth state_territory\n",
      "Topic 79: security agency card crime federal_police person police asio power attorney_general\n",
      "Topic 80: wong xenophon abetz supplementary cash cameron senator_interjecting moore relation expired\n",
      "Topic 81: department information public_service including agency guideline consideration charity provision answer_honourable\n",
      "Topic 82: right case provision court law agreement power human_right commission amendment\n",
      "Topic 83: industry coal country need company mine waste power_station going energy\n",
      "Topic 84: going know want come thing need actually green country like\n",
      "Topic 85: scheme support measure ensure reform fund state_territory funding provide commonwealth\n",
      "Topic 86: qantas airline system fielding plebiscite labor democrat change taxpayer service\n",
      "Topic 87: research animal trade country disease animal_welfare afghanistan indonesia sheep india\n",
      "Topic 88: plan commonwealth state environment process agreement water murray_darling basin first_agreed\n",
      "Topic 89: insurance private_health program insurer premium cost cover health_insurance fund service\n",
      "Topic 90: information online data internet privacy gambling material record access individual\n",
      "Topic 91: community work young program local support school service club electorate\n",
      "Topic 92: tenant offshore_detention leader_resume hick roxon myefo taiwan david_hick concluded_answer guantanamo\n",
      "Topic 93: veteran service veteran_affair member member_follows notice_given department_veteran veteran_family veteran_community affair\n",
      "Topic 94: climate_change country world green know need science fact environment policy\n",
      "Topic 95: consumer price product carbon market industry cost wine accc good\n",
      "Topic 96: health medicare system hospital private_health patient doctor health_care public_hospital australian\n",
      "Topic 97: farmer drought farm agriculture support industry rural dairy_farmer need business\n",
      "Topic 98: marriage firsthand relationship couple minister_department airport resolution sanction change iraq\n",
      "Topic 99: country world support force australian security right iraq know human_right\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for i in range(lda_model.num_topics):\n",
    "    print_topic(lda_model,i, n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}