assignments/assignment_3/flueckiger_KED2022_3_solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ff251679-e9ca-4a51-86ca-f36c42eee602",
   "metadata": {},
   "source": [
    "# Assignment 3 - Example Solution\n",
    "### Alex Flückiger\n",
    "### Seminar: The ABC of computational Text Analysis\n",
    "### University of Lucerne\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3302d348-b302-4bb3-b800-ddb23ab43cb1",
   "metadata": {},
   "source": [
    "## Preparational part (not part of assignment)\n",
    "\n",
    "The working directory is set to KED2022 instead to the directory where this Jupyter notebook is located.\n",
    "Thus, I can specify the pathes relative to the top folder KED2022. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "601db9ec-9b17-4d0b-812d-59af42b22304",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Old working directory: /home/alex/KED2022/assignments/assignment_3\n",
      "New working directory: /home/alex/KED2022\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(\"Old working directory:\", os.getcwd())\n",
    "os.chdir('../..')\n",
    "print(\"New working directory:\", os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c8c741d-452c-4b17-ac1b-e196af20e462",
   "metadata": {},
   "source": [
    "To run this notebook as script from the command line, you can export it as .py file via the menubar and save it to assignments/assignment_3/flueckiger_KED2022_3_solutions.py.\n",
    "\n",
    "Then, you can navigate to the KED2022 directory and call:\n",
    "\n",
    "```\n",
    "python assignments/assignment_3/flueckiger_KED2022_3_solutions.py\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae48c647-f274-412e-b24f-03c7f63d70e1",
   "metadata": {},
   "source": [
    "## Tasks of Assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c977bca8-26ff-4deb-ae82-515229310322",
   "metadata": {},
   "outputs": [],
   "source": [
    "# task 3: import the modules that are needed\n",
    "import textacy\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "df06d349-b87a-4661-a81f-cf0e88075427",
   "metadata": {},
   "outputs": [],
   "source": [
    "# task 4: create a corpus\n",
    "# define new function to read text from csv file (copied as given in assignment)\n",
    "\n",
    "def get_texts_from_csv(f_csv, text_column):\n",
    "\n",
    "    # read dataframe\n",
    "    df = pd.read_csv(f_csv)\n",
    "\n",
    "    # keep only documents that have text\n",
    "    filtered_df = df[df[text_column].notnull()]\n",
    "\n",
    "    # iterate over rows in dataframe\n",
    "    for idx, row in filtered_df.iterrows():\n",
    "\n",
    "        # read text and join lines (hard line-breaks)\n",
    "        text = row[text_column].replace(\"\\n\", \" \")\n",
    "\n",
    "        # use all columns as metadata, except the column with the actual text\n",
    "        metadata = row.to_dict()\n",
    "        del metadata[text_column]\n",
    "\n",
    "        yield (text, metadata)\n",
    "\n",
    "\n",
    "# set correct path relative to working directory (folder where you saved this script)\n",
    "f_csv = \"materials/data/dataset_speeches_federal_council_2019.csv\"\n",
    "\n",
    "# stream the csv-dataset by calling the function defined above\n",
    "texts = get_texts_from_csv(f_csv, text_column=\"Text\")\n",
    "\n",
    "# load German language model\n",
    "de = textacy.load_spacy_lang(\"de_core_news_sm\")\n",
    "\n",
    "# create a corpus with all the texts\n",
    "corpus_speeches = textacy.Corpus(de, data=texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eade345a-b0c2-49d4-afca-0077b319bd33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# task 5: two subcorpora\n",
    "# define two functions filtering by language and period\n",
    "# similar as the lambda functions shown in assignment, yet they may be simpler to understand\n",
    "\n",
    "def filter_func_pre(doc):\n",
    "    return doc._.meta.get(\"Sprache\") == \"de\" and doc._.meta.get(\"Jahr\") < 2000\n",
    "\n",
    "\n",
    "# greater-equal to include the year 2000\n",
    "def filter_func_post(doc):\n",
    "    return doc._.meta.get(\"Sprache\") == \"de\" and doc._.meta.get(\"Jahr\") >= 2000\n",
    "\n",
    "\n",
    "#########################################\n",
    "# ALTERNATIVE task 5: filter by gender instead of period\n",
    "# You just need to call these functions when creating the supcorpora below.\n",
    "# Everything else keeps the same.\n",
    "def filter_func_women(doc):\n",
    "    return (\n",
    "        doc._.meta.get(\"Sprache\") == \"de\"\n",
    "        and doc._.meta.get(\"Jahr\") >= 1999\n",
    "        and doc._.meta.get(\"Geschlecht\") >= \"f\"\n",
    "    )\n",
    "\n",
    "\n",
    "def filter_func_men(doc):\n",
    "    return (\n",
    "        doc._.meta.get(\"Sprache\") == \"de\"\n",
    "        and doc._.meta.get(\"Jahr\") >= 1999\n",
    "        and doc._.meta.get(\"Geschlecht\") >= \"m\"\n",
    "    )\n",
    "\n",
    "\n",
    "#########################################\n",
    "\n",
    "# create two new subcorpora after applying filter function\n",
    "subcorpus_pre = textacy.corpus.Corpus(de, data=corpus_speeches.get(filter_func_pre))\n",
    "subcorpus_post = textacy.corpus.Corpus(de, data=corpus_speeches.get(filter_func_post))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "094c0ef6-d59a-445e-b813-b40b7a3f826a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# documents in Subcorpus 1 (before 2000): 63\n",
      "# documents in Subcorpus 2 (as of 2000): 97\n"
     ]
    }
   ],
   "source": [
    "# task 6: print number of docs for both subcorpora\n",
    "print(\"# documents in Subcorpus 1 (before 2000):\", subcorpus_pre.n_docs)\n",
    "print(\"# documents in Subcorpus 2 (as of 2000):\", subcorpus_post.n_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e177581-c6e8-4b2a-afe2-904c58740590",
   "metadata": {},
   "outputs": [],
   "source": [
    "# task 7: export the vocabulary of both subcorpora into file\n",
    "\n",
    "# lowercased, unigram\n",
    "vocab_pre = subcorpus_pre.word_counts(\n",
    "    by=\"lower_\",\n",
    "    filter_stops=True,\n",
    "    filter_punct=True,\n",
    "    filter_nums=True,\n",
    "    weighting=\"freq\",  # get relative frequency instead of absolute\n",
    ")\n",
    "vocab_post = subcorpus_post.word_counts(\n",
    "    by=\"lower_\",\n",
    "    filter_stops=True,\n",
    "    filter_punct=True,\n",
    "    filter_nums=True,\n",
    "    weighting=\"freq\",\n",
    ")\n",
    "\n",
    "# sort vocabulary by descending frequency\n",
    "vocab_sorted_pre = sorted(vocab_pre.items(), key=lambda x: x[1], reverse=True)\n",
    "vocab_sorted_post = sorted(vocab_post.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "# write to file, one word and its frequency per line\n",
    "fname = \"assignments/assignment_3/vocab_frq_pre.txt\"\n",
    "with open(fname, \"w\") as f:\n",
    "    for word, frq in vocab_sorted_pre:\n",
    "        line = f\"{word}\\t{frq}\\n\"\n",
    "        f.write(line)\n",
    "\n",
    "fname = \"assignments/assignment_3/vocab_frq_post.txt\"\n",
    "with open(fname, \"w\") as f:\n",
    "    for word, frq in vocab_sorted_post:\n",
    "        line = f\"{word}\\t{frq}\\n\"\n",
    "        f.write(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d01373d-e24c-48eb-8618-bfdaad3b5cb4",
   "metadata": {},
   "source": [
    "### task 8: comparison of both vocabularies - changes? Which are the terms that are used most often?\n",
    "\n",
    "### Interpretation\n",
    "The top ten words look surprisingly similiar. Tradition seems to be reflected not only in the ritual itself but in the language of these speeches as well.\n",
    "\n",
    "At a second glance, there are interesting differences though:\n",
    "- speakers start to talk more about \"europa\" and \"EU\" after 2000\n",
    "- nowadays, the term \"Eidgenossenschaft\" is primarily used by right-wing people and got generally replaced by \"Schweiz\"\n",
    "- gender became important: greeting women as \"Damen\" besides \"Herren\" (notably, there are not only \"Schweizerinnen und Schweizer\")\n",
    "- \"Sicherheit\" becomes a hot topic after the relatively calm post-war period\n",
    "- increased talking about \"Kultur\", \"Identität\", \"Werte\". Who are we (no longer limited to Swiss people) in a globalized and multi-cultural world?\n",
    "\n",
    "To compare the vocabulary in a more systematic way than simple eye-balling,\n",
    "it helps to compute the relative change in frequency between the epochs.\n",
    "This goes beyond what we have touched upon in the seminar. Yet, it is worth to look at it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4e1fee8f-c8d5-46c7-a474-1a8b0584d8e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataframe for both subcorpora\n",
    "df_pre = pd.DataFrame(vocab_sorted_pre, columns=[\"term\", \"frq\"])\n",
    "df_post = pd.DataFrame(vocab_sorted_post, columns=[\"term\", \"frq\"])\n",
    "\n",
    "# merge them into a single dataset on the basis of term\n",
    "df = df_pre.merge(df_post, on=\"term\", how=\"inner\", suffixes=(\"_pre\", \"_post\"))\n",
    "\n",
    "# compute the relative difference and sort by it\n",
    "df[\"diff\"] = df[\"frq_post\"] - df[\"frq_pre\"]\n",
    "df.sort_values(\"diff\", inplace=True)\n",
    "\n",
    "# save the results as csv dataset\n",
    "fname = \"assignments/assignment_3/vocab_diff_periods.csv\"\n",
    "df.to_csv(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7b129818-214c-4a99-91f7-21c701030f36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>frq_pre</th>\n",
       "      <th>frq_post</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>feiern</td>\n",
       "      <td>0.000437</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>bevölkerung</td>\n",
       "      <td>0.000185</td>\n",
       "      <td>0.000477</td>\n",
       "      <td>0.000292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>kultur</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>0.000536</td>\n",
       "      <td>0.000301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>949</th>\n",
       "      <td>bürgerinnen</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.000386</td>\n",
       "      <td>0.000319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927</th>\n",
       "      <td>erfolg</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.000392</td>\n",
       "      <td>0.000325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>wohlstand</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.000595</td>\n",
       "      <td>0.000326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1212</th>\n",
       "      <td>identität</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.000392</td>\n",
       "      <td>0.000342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>braucht</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.000342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>herren</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>0.000549</td>\n",
       "      <td>0.000381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1260</th>\n",
       "      <td>wichtig</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.000438</td>\n",
       "      <td>0.000388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>gemeinsam</td>\n",
       "      <td>0.000302</td>\n",
       "      <td>0.000693</td>\n",
       "      <td>0.000391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>weg</td>\n",
       "      <td>0.000487</td>\n",
       "      <td>0.000883</td>\n",
       "      <td>0.000395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>gesellschaft</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>0.000398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>werte</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.000706</td>\n",
       "      <td>0.000437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734</th>\n",
       "      <td>damen</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000530</td>\n",
       "      <td>0.000446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>sicherheit</td>\n",
       "      <td>0.000286</td>\n",
       "      <td>0.000883</td>\n",
       "      <td>0.000597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>schweizer</td>\n",
       "      <td>0.001059</td>\n",
       "      <td>0.001857</td>\n",
       "      <td>0.000798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>menschen</td>\n",
       "      <td>0.001025</td>\n",
       "      <td>0.001896</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>land</td>\n",
       "      <td>0.002571</td>\n",
       "      <td>0.003818</td>\n",
       "      <td>0.001247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>schweiz</td>\n",
       "      <td>0.002554</td>\n",
       "      <td>0.006989</td>\n",
       "      <td>0.004435</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              term   frq_pre  frq_post      diff\n",
       "82          feiern  0.000437  0.000719  0.000282\n",
       "305    bevölkerung  0.000185  0.000477  0.000292\n",
       "220         kultur  0.000235  0.000536  0.000301\n",
       "949    bürgerinnen  0.000067  0.000386  0.000319\n",
       "927         erfolg  0.000067  0.000392  0.000325\n",
       "195      wohlstand  0.000269  0.000595  0.000326\n",
       "1212     identität  0.000050  0.000392  0.000342\n",
       "355        braucht  0.000168  0.000510  0.000342\n",
       "344         herren  0.000168  0.000549  0.000381\n",
       "1260       wichtig  0.000050  0.000438  0.000388\n",
       "158      gemeinsam  0.000302  0.000693  0.000391\n",
       "66             weg  0.000487  0.000883  0.000395\n",
       "185   gesellschaft  0.000269  0.000667  0.000398\n",
       "194          werte  0.000269  0.000706  0.000437\n",
       "734          damen  0.000084  0.000530  0.000446\n",
       "180     sicherheit  0.000286  0.000883  0.000597\n",
       "14       schweizer  0.001059  0.001857  0.000798\n",
       "15        menschen  0.001025  0.001896  0.000871\n",
       "0             land  0.002571  0.003818  0.001247\n",
       "1          schweiz  0.002554  0.006989  0.004435"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show terms with biggest increase wtr to relative frequency\n",
    "df.tail(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "eb33e888-a2b6-46c3-a8e5-9241b7c657fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>frq_pre</th>\n",
       "      <th>frq_post</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unseres</td>\n",
       "      <td>0.002268</td>\n",
       "      <td>0.001013</td>\n",
       "      <td>-0.001255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>eidgenossen</td>\n",
       "      <td>0.001126</td>\n",
       "      <td>0.000216</td>\n",
       "      <td>-0.000910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>eidgenossenschaft</td>\n",
       "      <td>0.001210</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>-0.000896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>landes</td>\n",
       "      <td>0.001663</td>\n",
       "      <td>0.001020</td>\n",
       "      <td>-0.000644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>staat</td>\n",
       "      <td>0.001277</td>\n",
       "      <td>0.000641</td>\n",
       "      <td>-0.000636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>unsern</td>\n",
       "      <td>0.000639</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>-0.000625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>probleme</td>\n",
       "      <td>0.000891</td>\n",
       "      <td>0.000268</td>\n",
       "      <td>-0.000622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>bund</td>\n",
       "      <td>0.000907</td>\n",
       "      <td>0.000288</td>\n",
       "      <td>-0.000620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>volk</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>0.000497</td>\n",
       "      <td>-0.000612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>aufgaben</td>\n",
       "      <td>0.000655</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>-0.000551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>gemeinschaft</td>\n",
       "      <td>0.000924</td>\n",
       "      <td>0.000379</td>\n",
       "      <td>-0.000545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>schweizervolk</td>\n",
       "      <td>0.000538</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>-0.000518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>willen</td>\n",
       "      <td>0.000790</td>\n",
       "      <td>0.000275</td>\n",
       "      <td>-0.000515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>schweizerischen</td>\n",
       "      <td>0.000639</td>\n",
       "      <td>0.000124</td>\n",
       "      <td>-0.000514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>innern</td>\n",
       "      <td>0.000538</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>-0.000498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>vaterland</td>\n",
       "      <td>0.000538</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>-0.000498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>kraft</td>\n",
       "      <td>0.000790</td>\n",
       "      <td>0.000294</td>\n",
       "      <td>-0.000496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>abend</td>\n",
       "      <td>0.000605</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>-0.000487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>frieden</td>\n",
       "      <td>0.000840</td>\n",
       "      <td>0.000366</td>\n",
       "      <td>-0.000474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>geist</td>\n",
       "      <td>0.000622</td>\n",
       "      <td>0.000163</td>\n",
       "      <td>-0.000458</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 term   frq_pre  frq_post      diff\n",
       "2             unseres  0.002268  0.001013 -0.001255\n",
       "11        eidgenossen  0.001126  0.000216 -0.000910\n",
       "10  eidgenossenschaft  0.001210  0.000314 -0.000896\n",
       "5              landes  0.001663  0.001020 -0.000644\n",
       "9               staat  0.001277  0.000641 -0.000636\n",
       "38             unsern  0.000639  0.000013 -0.000625\n",
       "23           probleme  0.000891  0.000268 -0.000622\n",
       "20               bund  0.000907  0.000288 -0.000620\n",
       "13               volk  0.001109  0.000497 -0.000612\n",
       "34           aufgaben  0.000655  0.000105 -0.000551\n",
       "18       gemeinschaft  0.000924  0.000379 -0.000545\n",
       "54      schweizervolk  0.000538  0.000020 -0.000518\n",
       "26             willen  0.000790  0.000275 -0.000515\n",
       "36    schweizerischen  0.000639  0.000124 -0.000514\n",
       "51             innern  0.000538  0.000039 -0.000498\n",
       "53          vaterland  0.000538  0.000039 -0.000498\n",
       "27              kraft  0.000790  0.000294 -0.000496\n",
       "42              abend  0.000605  0.000118 -0.000487\n",
       "24            frieden  0.000840  0.000366 -0.000474\n",
       "39              geist  0.000622  0.000163 -0.000458"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show terms with biggest decrease wtr to relative frequency\n",
    "df.head(20)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8bf0feaab3ef5ef83b3ac16af7ed048a6f87c509d64f2a500fc7c37eb089bc92"
  },
  "kernelspec": {
   "display_name": "ked2022",
   "language": "python",
   "name": "ked2022"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}