topics/pandas/word_frequency.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../../') # To be able to reach 'Repos' folder\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Books:\n",
      "The Brothers Karamazov.txt\n",
      "The Idiot.txt\n",
      "The Possessed.txt\n",
      "Poor Folk.txt\n",
      "Crime and Punishment.txt\n",
      "Notes from the Underground.txt\n",
      "White Nights and Other Stories.txt\n",
      "Short Stories.txt\n",
      "The House of the Dead.txt\n",
      "The Gambler.txt\n",
      "\n",
      "Evolution of the text:\n",
      "The Brothers Karamazov  PART I     Book I. The History Of A Family     Chapter I. Fyodor Pavlov\n",
      "the brothers karamazov  part i     book i. the history of a family     chapter i. fyodor pavlov\n",
      "the brothers karamazov  part i     book i the history of a family     chapter i fyodor pavlovit\n"
     ]
    }
   ],
   "source": [
    "dataset_path = Path.cwd().parent.parent.parent / 'py4ML' / 'datasets' / 'dostoyevski'\n",
    "book_names = []\n",
    "[book_names.append(item) for item in dataset_path.iterdir() if item.suffix == '.txt']\n",
    "text = []\n",
    "print('Books:')\n",
    "for book_name in book_names: file = open(book_name, 'r'); text.append(file.read()); file.close(); print(book_name.name)\n",
    "print('\\nEvolution of the text:')\n",
    "text = ' '.join(text)\n",
    "text = text.replace('\\n', ' ')\n",
    "def print_text(text): print(f'{text[:95]}')\n",
    "print_text(text)\n",
    "text = text.lower()\n",
    "print_text(text)\n",
    "for char in sorted(set(text)): text = text if char in 'abcdefghijklmnopqrstuvwxyz ' else text.replace(char, '')\n",
    "print_text(text)\n",
    "del dataset_path, book_names, book_name, file, char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = \\\n",
    "    \"\"\"contentment, contentment, gregarious, gullible, pompous, ruthless, tactful, tactless, witty,\n",
    "pot, dump, strip, hook, lay, pound, retain, strain, collar, rid, spite, clerk, prompt, boot, decent,\n",
    "neat, resort, rip, sail, tale, attorney, carpenter, jeweller, mechanic, physician, undertaker,\n",
    "affection, affectionate, aggravation, anguish, defeat, defiance, delight, dismay, wrath,\n",
    "wonder, scorn, pity, malice, lust, longing, hostility, hatred, grief, greed, fury, layoff,\n",
    "accolade, circumvent, dearth, ludicrous, mettle, morose, mundane, scrutinise, ignominious,\n",
    "ignominy, incidental, indefatigable, ingratiate, inscrutable, irksome, irresolute, jibe,\n",
    "obviate, obdurate, nettle, divulge, discursive, disconcert, diffident, timid, zealot, zenith,\n",
    "edict, effete, accede, entreat, expatiate, fatuous, gibe, ensue, accord, alacrity, askance,\n",
    "avarice, bane, banter, belie, bland, bucolic, bulwark, waive, vilify, vestige, turgid, stupendous,\n",
    "simper, glib, renegade, redolent, rebuke, querulous, platitude, permeate, peremptory, credence,\n",
    "corpulent, copious, construe, cogent, coalesce, circumspect, candid, callous, wander, venture,\n",
    "tribe, treaty, trail, testimony, testify, summit, sue, subsequent, stroke, stare, snap, sigh,\n",
    "shrug, settlement, seize, rifle, quarterback, prominent, portray, porch, pine, pile, patch, pant,\n",
    "pan, pale, opponent, nod, literary, legislation, lawsuit, lawn, inquiry, incorporate, incentive,\n",
    "grant, grain, gaze, flee, fade, dismiss, deputy, deficit, conviction, congressional, buck, barrel,\n",
    "ally, alliance, aide, capricious, vigilant, pleading, endeavor, errand, appeal, appealing, frankly,\n",
    "fame, glory, acclaim, plague, fierce, roar, utterly, wicked, chop, bishop, proclaim, benevolent,\n",
    "excavation, reconcile, rumor, petrified, oblivion, indulge, debunk, prodigy, plunder, oblivious,\n",
    "allegation, alibi, indecisive, vile, condone, raid, riot, smuggling, strangle, bribe, legitimate,\n",
    "soar, plunge, recuperate, trunk, stagnant, hassle, sore, strand, deleterious, stunned,\n",
    "devastate, thrill, overwhelm, bewildered, frustrated, grumpy, superstition, entitled, petition,\n",
    "whine, lend, faint, charm, withhold, firm, prosecutor, delude, wig, bewitch, benign, adept,\n",
    "expedite, snooze, discretion, smother, provision, commence, escalate, exclamation, amend, perpetual,\n",
    "logistics, entitle, subsidiary, thereafter, demote, complimentary, predecessor, lurk, mischief,\n",
    "prompt, dignity, stoop, incite, mustard, mighty, redemption, gloom, glooming, condemn, commodity,\n",
    "hypocrite, fidelity, infidelity, loyalty, stubborn, arrogant, bold, nascent, alleviate, foster,\n",
    "shatter, nasty, congruent, dire, congruence, incongruent, precarious, off-putting, instalment,\n",
    "verdict, degrease, rust, inapt, customs, rip off, bowel, rattle, falcon, prologue, needy, enfold,\n",
    "benevolence, bestow, bliss, blithe, blockbuster, bona fide, cushion, legume, stir, vague, waist,\n",
    "dew, cot, groan, maid, hustle, tame, fondle, vicious, agony, farewell, sicken, hone, foreclose,\n",
    "cynic, cynical, tender, ambivalence, embark, endeavour, arrogance, eager, ascribe, resentment,\n",
    "palate, spacious, sage, nip, assert, assertion, vindication, tentatively, mesmerise, cloak, stew,\n",
    "boar, pulp, neat, lump, set off, interest, flutter, cult, haze, Algeria, raven, morocco, omnipresent,\n",
    "invoice, gravy, barley, leek, cling, thyme, lodge, otter, voucher, loan, oath, hitchhike, succeed,\n",
    "successor, dandruff, jaw, grace, hospice, rhubarb, unabridged, nausea, rhino, mole, rash, vigorous,\n",
    "clumsy, mercenary, pea, afflict, shiver, seizure, porridge, stink, taint, bust, muster, incense,\n",
    "rinse, hoarding, salient, cease, sewerage, fin, rasp, usurer, socket, sow, porch, shed, stale,\n",
    "corny, sobriety, mate, raisin, clerical, celeriac, shutter, valve, beetroot, lactating, laxative,\n",
    "brisk, bloat, patio, kudos, haul, odour, fabric, seal, rig, grind, clench, roam, snarl, formidable,\n",
    "solidarity, salvation, ditch, legacy, burrow, mastic, florist, parsley, wink, rod, fern, grease, grime,\n",
    "rope, assertive, dispense, aggregate, hilarious, compassion, pour, remnant, hedgehog, comb, pole,\n",
    "remittance, concierge, turmeric, council, cul de sac, rug, tipsy, daze, intriguing, pondering, perplex,\n",
    "alleviate, setback, excel, gut, contentious, strive, virtually, chameleon, flattery, shortcoming, rein,\n",
    "stamina, whimsical, lousy, impromptu, rigorous, enthusiastic, sluggish, utmost, retain, modest, blunt,\n",
    "composure, quirk, reprimand, preach, humble, intimidated, vain, subtlety\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "studied = \\\n",
    "    \"\"\"chore, deputy, resent, gooey, juggle, abduct, taser, rapport, chuckle, shiver,\n",
    "facade, outlandish, slur, sergeant, rattle, allegedly, dazed, chilling, dispatcher,\n",
    "accomplice, unwittingly, plead, perpetrator, culprit, oblivion, petrified, plunder,\n",
    "prodigy, debunk, poke, indulge, salient, oblivious, embark, allegation, wander,\n",
    "wonder, seize, appeal, sigh, alibi, sail, accord, lay, tear, lustful, stun gun, spite,\n",
    "pale, nonchalant, clerk, interest, prosecutor, utterly, dignity, conviction, pole,\n",
    "demeanour, grandiose, succumb, vindictive, retaliate, mould, snap, scoff, felony,\n",
    "misdemeanour, timid, bleach, alleviate, vain, stutter, tranquil, compassion,\n",
    "compassionate, vitality, conscience, vigor, contentment, discontent, prosperity,\n",
    "prosperous, detach, defeat, grace, abundant, lack, aside, petty, affirmation,\n",
    "relentless, condemn, promiscuous, furnace, lavish, arson, peason, tender, diligent,\n",
    "meticulous, serendipity, errand, mundane, condolence, condo, arduous, whereabouts,\n",
    "obnoxious, odour, tedious, generalist, menace, abominable, dull, feat, improvise,\n",
    "dissect, traverse, heinous, deterrent, baffle, harrowing, solicit, atrocious, whim,\n",
    "undertaking, hypocrite, ruminate, funnel, entail, vigilant, decent, agony, frankly,\n",
    "vile, daunting, temper, venture, composure, impediment, legit\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>contentment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>contentment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gregarious</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gullible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pompous</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          word\n",
       "0  contentment\n",
       "1  contentment\n",
       "2   gregarious\n",
       "3     gullible\n",
       "4      pompous"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = words.replace('\\n', ' ')\n",
    "words = words.split(', ')\n",
    "df = pd.DataFrame({'word': words})\n",
    "del words\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>contentment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gregarious</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gullible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pompous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ruthless</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          word\n",
       "0  contentment\n",
       "2   gregarious\n",
       "3     gullible\n",
       "4      pompous\n",
       "5     ruthless"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop_duplicates()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>contentment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gregarious</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gullible</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pompous</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ruthless</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          word  freq\n",
       "0  contentment     1\n",
       "2   gregarious     0\n",
       "3     gullible     0\n",
       "4      pompous     7\n",
       "5     ruthless     2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['freq'] = (' ' + df['word'] + ' ').apply(text.count) + (' ' + df['word'] + 's ').apply(text.count)\n",
    "del text\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spite</td>\n",
       "      <td>417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lay</td>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pale</td>\n",
       "      <td>304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>interest</td>\n",
       "      <td>296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>clerk</td>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       word  freq\n",
       "0     spite   417\n",
       "1       lay   334\n",
       "2      pale   304\n",
       "3  interest   296\n",
       "4     clerk   193"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sort_values('freq', ascending=False, ignore_index=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pity</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>hatred</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>grief</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>delight</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>boot</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       word  freq\n",
       "9      pity   149\n",
       "12   hatred   138\n",
       "13    grief   122\n",
       "14  delight   122\n",
       "15     boot   120"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "studied = studied.replace('\\n', ' ')\n",
    "studied = studied.split(', ')\n",
    "df = df[~df['word'].isin(studied)]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9            pity\n",
      "12         hatred\n",
      "13          grief\n",
      "14        delight\n",
      "15           boot\n",
      "16           fury\n",
      "17           shed\n",
      "18         utmost\n",
      "19       literary\n",
      "20          nasty\n",
      "21        set off\n",
      "22            rid\n",
      "24         humble\n",
      "25        anguish\n",
      "26    exclamation\n",
      "27       farewell\n",
      "29          eager\n",
      "30          faint\n",
      "31           firm\n",
      "33         stroke\n",
      "35          stare\n",
      "37          lodge\n",
      "39          vague\n",
      "40      affection\n",
      "41        inquiry\n",
      "42           tale\n",
      "43     resentment\n",
      "44           gaze\n",
      "45           bold\n",
      "46          glory\n",
      "47           stir\n",
      "48          wrath\n",
      "49         wicked\n",
      "50         modest\n",
      "51         dismay\n",
      "52        longing\n",
      "53          cease\n",
      "54        succeed\n",
      "57          cloak\n",
      "59     bewildered\n",
      "60            rod\n",
      "63         collar\n",
      "64          pound\n",
      "66       mischief\n",
      "67           maid\n",
      "69           pour\n",
      "70            pan\n",
      "71          witty\n",
      "72          trunk\n",
      "74         morose\n",
      "Name: word, dtype: object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/l6/2q9c0f9127bd3ydvrgk19ncw0000gn/T/ipykernel_68202/3363676688.py:1: FutureWarning: The behavior of `series[i:j]` with an integer-dtype index is deprecated. In a future version, this will be treated as *label-based* indexing, consistent with e.g. `series[i]` lookups. To retain the old behavior, use `series.iloc[i:j]`. To get the future behavior, use `series.loc[i:j]`.\n",
      "  print(df['word'][:50])\n"
     ]
    }
   ],
   "source": [
    "print(df['word'][:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_it_in_my_list(word): return df['word'].eq(word).any()\n",
    "def have_i_studied(word): return word in studied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_it_in_my_list(word) = False\n",
      "have_i_studied(word) = True\n"
     ]
    }
   ],
   "source": [
    "word = 'legit'\n",
    "print(f\"{is_it_in_my_list(word) = }\")\n",
    "print(f\"{have_i_studied(word) = }\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyP1FeOK7v/egWCsVIKGnu/k",
   "name": "Spectrogram.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3.10.6 ('study')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "29d06b973d1ddb34db3279b24f9b5152402e688db937648b736e855ec4de60c0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}