ecce.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ecce - \"Behold\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table of Contents\n",
    "\n",
    "1. Overview\n",
    "2. English Standard Version (data processing)\n",
    "3. Nave's Topical Index\n",
    "  - Data preprocessing\n",
    "  - Exploratory Data Analysis\n",
    "  - Modeling\n",
    "4. Treasury of Scripture Knowledge\n",
    "  - Data preprocessing\n",
    "  - Exploratory Data Analysis\n",
    "  - Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from ecce.utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## English Standard Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ecce/data/ESV.json']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glob('ecce/data/ESV*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Loading ESV JSON...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('Judges',\n",
       "  '11',\n",
       "  '24',\n",
       "  'Will you not possess what Chemosh your god gives you to possess? And all that the LORD our God has dispossessed before us, we will possess.'),\n",
       " ('Judges',\n",
       "  '11',\n",
       "  '25',\n",
       "  'Now are you any better than Balak the son of Zippor, king of Moab? Did he ever contend against Israel, or did he ever go to war with them?')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ecce.esv as esv\n",
    "import ecce.reference as reference\n",
    "\n",
    "list(esv.flattened_verses())[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'In the beginning, God created the heavens and the earth.'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "esv.text(reference.init('Genesis', 1, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nave's Topical Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ecce/data/nave/sample_sql.txt',\n",
       " 'ecce/data/nave/altered_nave.dat',\n",
       " 'ecce/data/nave/topicxref.txt',\n",
       " 'ecce/data/nave/subtopics.txt',\n",
       " 'ecce/data/nave/topics.txt',\n",
       " 'ecce/data/nave/nave.dat',\n",
       " 'ecce/data/nave/readme.txt',\n",
       " 'ecce/data/nave/categories.txt']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glob('ecce/data/nave/[atsnrc]*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_key</th>\n",
       "      <th>category_key</th>\n",
       "      <th>subtopic_key</th>\n",
       "      <th>subtopic_text</th>\n",
       "      <th>sort_order_subtopic</th>\n",
       "      <th>source_topic_key</th>\n",
       "      <th>reference_list</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>category_text</th>\n",
       "      <th>sort_order_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2146356859</td>\n",
       "      <td>-2108238392</td>\n",
       "      <td>0</td>\n",
       "      <td>General reference(s) to this category</td>\n",
       "      <td>0</td>\n",
       "      <td>$$T0001327</td>\n",
       "      <td>Ex32:19,25</td>\n",
       "      <td>DANCING</td>\n",
       "      <td>Idolatrous</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2146356859</td>\n",
       "      <td>-1508129049</td>\n",
       "      <td>0</td>\n",
       "      <td>General reference(s) to this category</td>\n",
       "      <td>0</td>\n",
       "      <td>$$T0001327</td>\n",
       "      <td>Mt14:6; Mr6:22</td>\n",
       "      <td>DANCING</td>\n",
       "      <td>Herodias dances in the presence of Herod (Anti...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2146356859</td>\n",
       "      <td>696243684</td>\n",
       "      <td>0</td>\n",
       "      <td>General reference(s) to this category</td>\n",
       "      <td>0</td>\n",
       "      <td>$$T0001327</td>\n",
       "      <td>Ex15:20; Ex32:19; Jud11:34; Jud21:19-21; 1Sa18...</td>\n",
       "      <td>DANCING</td>\n",
       "      <td>General scriptures concerning</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2145736290</td>\n",
       "      <td>-148185818</td>\n",
       "      <td>0</td>\n",
       "      <td>General reference(s) to this category</td>\n",
       "      <td>0</td>\n",
       "      <td>$$T0001234</td>\n",
       "      <td>Le11:17; De14:17; Isa34:11; Zep2:14</td>\n",
       "      <td>CORMORANT</td>\n",
       "      <td>A bird forbidden as food</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-2145523250</td>\n",
       "      <td>1308325410</td>\n",
       "      <td>0</td>\n",
       "      <td>General reference(s) to this category</td>\n",
       "      <td>0</td>\n",
       "      <td>$$T0000957</td>\n",
       "      <td>Ac8:27</td>\n",
       "      <td>CANDACE</td>\n",
       "      <td>Queen of Ethiopia</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    topic_key  category_key  subtopic_key  \\\n",
       "0 -2146356859   -2108238392             0   \n",
       "1 -2146356859   -1508129049             0   \n",
       "2 -2146356859     696243684             0   \n",
       "3 -2145736290    -148185818             0   \n",
       "4 -2145523250    1308325410             0   \n",
       "\n",
       "                           subtopic_text  sort_order_subtopic  \\\n",
       "0  General reference(s) to this category                    0   \n",
       "1  General reference(s) to this category                    0   \n",
       "2  General reference(s) to this category                    0   \n",
       "3  General reference(s) to this category                    0   \n",
       "4  General reference(s) to this category                    0   \n",
       "\n",
       "  source_topic_key                                     reference_list  \\\n",
       "0       $$T0001327                                         Ex32:19,25   \n",
       "1       $$T0001327                                     Mt14:6; Mr6:22   \n",
       "2       $$T0001327  Ex15:20; Ex32:19; Jud11:34; Jud21:19-21; 1Sa18...   \n",
       "3       $$T0001234                Le11:17; De14:17; Isa34:11; Zep2:14   \n",
       "4       $$T0000957                                             Ac8:27   \n",
       "\n",
       "  topic_name                                      category_text  \\\n",
       "0    DANCING                                         Idolatrous   \n",
       "1    DANCING  Herodias dances in the presence of Herod (Anti...   \n",
       "2    DANCING                      General scriptures concerning   \n",
       "3  CORMORANT                           A bird forbidden as food   \n",
       "4    CANDACE                                  Queen of Ethiopia   \n",
       "\n",
       "   sort_order_category  \n",
       "0                  3.0  \n",
       "1                  2.0  \n",
       "2                  1.0  \n",
       "3                  1.0  \n",
       "4                  1.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ecce.nave as nave\n",
    "import ecce.model.nave.data as nave_data\n",
    "\n",
    "nave.df().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Le11:17; De14:17; Isa34:11; Zep2:14'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.df().at[3, 'reference_list']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Reference(book='Leviticus', chapter=11, verse=17),\n",
       " Reference(book='Deuteronomy', chapter=14, verse=17),\n",
       " Reference(book='Isaiah', chapter=34, verse=11),\n",
       " Reference(book='Zephaniah', chapter=2, verse=14)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.parse(nave.df().at[3, 'reference_list'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(Reference(book='1 Corinthians', chapter=9, verse=13),\n",
       "  {'topic_key': -2054210176,\n",
       "   'category_key': -529042571,\n",
       "   'subtopic_key': 1255183917,\n",
       "   'subtopic_text': 'In supporting himself',\n",
       "   'sort_order_subtopic': 2,\n",
       "   'source_topic_key': '$$T0001742',\n",
       "   'reference_list': '1Co9:7-23',\n",
       "   'topic_name': 'EVIL',\n",
       "   'category_text': 'INSTANCES OF',\n",
       "   'sort_order_category': 2.0}),\n",
       " (Reference(book='1 Corinthians', chapter=9, verse=14),\n",
       "  {'topic_key': -2054210176,\n",
       "   'category_key': -529042571,\n",
       "   'subtopic_key': 1255183917,\n",
       "   'subtopic_text': 'In supporting himself',\n",
       "   'sort_order_subtopic': 2,\n",
       "   'source_topic_key': '$$T0001742',\n",
       "   'reference_list': '1Co9:7-23',\n",
       "   'topic_name': 'EVIL',\n",
       "   'category_text': 'INSTANCES OF',\n",
       "   'sort_order_category': 2.0})]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.init()[1000:1002]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Skipping 73 verses with no assigned topics.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book</th>\n",
       "      <th>chapter</th>\n",
       "      <th>verse</th>\n",
       "      <th>topics</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1004</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>[PRIDE, HUMILITY, WISDOM, PARADOX]</td>\n",
       "      <td>Let no one deceive himself. If anyone among yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>[QUOTATIONS, ALLUSIONS, WISDOM, IGNORANCE]</td>\n",
       "      <td>For the wisdom of this world is folly with God...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1006</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>[WISDOM, ALLUSIONS, QUOTATIONS, VANITY, GOD, H...</td>\n",
       "      <td>and again, \"The Lord knows the thoughts of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>[CONFIDENCE, Christian, RIGHTEOUS, DEATH, MINI...</td>\n",
       "      <td>So let no one boast in men. For all things are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1008</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>[DEATH, GOD, RIGHTEOUS]</td>\n",
       "      <td>whether Paul or Apollos or Cephas or the world...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1009</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>[JESUS, RIGHTEOUS, DEATH, THE CHRIST, GOD]</td>\n",
       "      <td>and you are Christ's, and Christ is God's.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1010</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[ZEAL, Christian, STEWARD, MINISTER, SERVANT]</td>\n",
       "      <td>This is how one should regard us, as servants ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1011</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>[FAITHFULNESS, ZEAL, Christian, STEWARD, MINIS...</td>\n",
       "      <td>Moreover, it is required of stewards that they...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1012</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>[ZEAL]</td>\n",
       "      <td>But with me it is a very small thing that I sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1013</th>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>[JESUS, THE CHRIST, ZEAL]</td>\n",
       "      <td>I am not aware of anything against myself, but...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               book  chapter  verse  \\\n",
       "1004  1 Corinthians        3     18   \n",
       "1005  1 Corinthians        3     19   \n",
       "1006  1 Corinthians        3     20   \n",
       "1007  1 Corinthians        3     21   \n",
       "1008  1 Corinthians        3     22   \n",
       "1009  1 Corinthians        3     23   \n",
       "1010  1 Corinthians        4      1   \n",
       "1011  1 Corinthians        4      2   \n",
       "1012  1 Corinthians        4      3   \n",
       "1013  1 Corinthians        4      4   \n",
       "\n",
       "                                                 topics  \\\n",
       "1004                 [PRIDE, HUMILITY, WISDOM, PARADOX]   \n",
       "1005         [QUOTATIONS, ALLUSIONS, WISDOM, IGNORANCE]   \n",
       "1006  [WISDOM, ALLUSIONS, QUOTATIONS, VANITY, GOD, H...   \n",
       "1007  [CONFIDENCE, Christian, RIGHTEOUS, DEATH, MINI...   \n",
       "1008                            [DEATH, GOD, RIGHTEOUS]   \n",
       "1009         [JESUS, RIGHTEOUS, DEATH, THE CHRIST, GOD]   \n",
       "1010      [ZEAL, Christian, STEWARD, MINISTER, SERVANT]   \n",
       "1011  [FAITHFULNESS, ZEAL, Christian, STEWARD, MINIS...   \n",
       "1012                                             [ZEAL]   \n",
       "1013                          [JESUS, THE CHRIST, ZEAL]   \n",
       "\n",
       "                                                   text  \n",
       "1004  Let no one deceive himself. If anyone among yo...  \n",
       "1005  For the wisdom of this world is folly with God...  \n",
       "1006  and again, \"The Lord knows the thoughts of the...  \n",
       "1007  So let no one boast in men. For all things are...  \n",
       "1008  whether Paul or Apollos or Cephas or the world...  \n",
       "1009         and you are Christ's, and Christ is God's.  \n",
       "1010  This is how one should regard us, as servants ...  \n",
       "1011  Moreover, it is required of stewards that they...  \n",
       "1012  But with me it is a very small thing that I sh...  \n",
       "1013  I am not aware of anything against myself, but...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave_data.frame().iloc[1000:1010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filtering topics (verses >= 30) changes count from 4398 to 853\n"
     ]
    }
   ],
   "source": [
    "from funcy import flatten, compose\n",
    "\n",
    "topic_count = compose(len, set, flatten, attr('topics'))\n",
    "\n",
    "raw_length = topic_count(nave_data.frame())\n",
    "filtered_length = topic_count(nave_data.filtered_frame(min_per_topic=30))\n",
    "\n",
    "print(f'Filtering topics (verses >= 30) changes count from {raw_length} to {filtered_length}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Topics per Verse\n",
      "###################################################################\n",
      "████████████                                        1011  1 topic  \n",
      "██████████████████████████                          2192  2 topics \n",
      "███████████████████████████████████████             3238  3 topics \n",
      "████████████████████████████████████████████████    3966  4 topics \n",
      "██████████████████████████████████████████████████  4063  5 topics \n",
      "██████████████████████████████████████████████      3755  6 topics \n",
      "███████████████████████████████████████             3176  7 topics \n",
      "████████████████████████████████                    2631  8 topics \n",
      "███████████████████████                             1944  9 topics \n",
      "█████████████████                                   1443  10 topics\n",
      "████████████                                        1033  11 topics\n",
      "█████████                                            778  12 topics\n",
      "██████                                               536  13 topics\n",
      "████                                                 394  14 topics\n",
      "███                                                  296  15 topics\n",
      "██                                                   192  16 topics\n",
      "█                                                    132  17 topics\n",
      "█                                                     95  18 topics\n",
      "                                                      38  19 topics\n",
      "                                                      22  20 topics\n",
      "                                                      11  21 topics\n",
      "                                                       9  22 topics\n",
      "                                                       5  23 topics\n",
      "                                                       1  24 topics\n",
      "                                                       1  25 topics\n",
      "                                                       4  26 topics\n",
      "                                                       3  27 topics\n"
     ]
    }
   ],
   "source": [
    "nave_data.print_topic_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_name</th>\n",
       "      <th>verse_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3536</th>\n",
       "      <td>GOD</td>\n",
       "      <td>6047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3968</th>\n",
       "      <td>JESUS</td>\n",
       "      <td>5069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3137</th>\n",
       "      <td>THE CHRIST</td>\n",
       "      <td>5069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>ISRAEL</td>\n",
       "      <td>5008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062</th>\n",
       "      <td>PSALMS</td>\n",
       "      <td>2738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3637</th>\n",
       "      <td>DAVID</td>\n",
       "      <td>2336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938</th>\n",
       "      <td>MINISTER</td>\n",
       "      <td>2220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3679</th>\n",
       "      <td>CHURCH</td>\n",
       "      <td>2198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2799</th>\n",
       "      <td>Christian</td>\n",
       "      <td>2155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4079</th>\n",
       "      <td>ADVERSITIES</td>\n",
       "      <td>1856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>AFFLICTIONS</td>\n",
       "      <td>1856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1186</th>\n",
       "      <td>SIN</td>\n",
       "      <td>1714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>WICKED PEOPLE</td>\n",
       "      <td>1551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1846</th>\n",
       "      <td>RULERS</td>\n",
       "      <td>1513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3711</th>\n",
       "      <td>FAITH</td>\n",
       "      <td>1487</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         topic_name  verse_count\n",
       "3536            GOD         6047\n",
       "3968          JESUS         5069\n",
       "3137     THE CHRIST         5069\n",
       "836          ISRAEL         5008\n",
       "1062         PSALMS         2738\n",
       "3637          DAVID         2336\n",
       "1938       MINISTER         2220\n",
       "3679         CHURCH         2198\n",
       "2799      Christian         2155\n",
       "4079    ADVERSITIES         1856\n",
       "636     AFFLICTIONS         1856\n",
       "1186            SIN         1714\n",
       "199   WICKED PEOPLE         1551\n",
       "1846         RULERS         1513\n",
       "3711          FAITH         1487"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave_data.verse_counts().sort_values(by='verse_count', ascending=False).iloc[0:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>verse</th>\n",
       "      <th>topic_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24276</th>\n",
       "      <td>Matthew 27:24</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24257</th>\n",
       "      <td>Matthew 27:5</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7036</th>\n",
       "      <td>Acts 26:18</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24692</th>\n",
       "      <td>Nehemiah 8:7</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3748</th>\n",
       "      <td>2 Chronicles 17:8</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24689</th>\n",
       "      <td>Nehemiah 8:4</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24256</th>\n",
       "      <td>Matthew 27:4</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24255</th>\n",
       "      <td>Matthew 27:3</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5066</th>\n",
       "      <td>2 Kings 19:37</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5127</th>\n",
       "      <td>2 Kings 22:14</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   verse  topic_count\n",
       "24276      Matthew 27:24           27\n",
       "24257       Matthew 27:5           27\n",
       "7036          Acts 26:18           27\n",
       "24692       Nehemiah 8:7           26\n",
       "3748   2 Chronicles 17:8           26\n",
       "24689       Nehemiah 8:4           26\n",
       "24256       Matthew 27:4           26\n",
       "24255       Matthew 27:3           25\n",
       "5066       2 Kings 19:37           24\n",
       "5127       2 Kings 22:14           23"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave_data.topic_counts().sort_values(by='topic_count', ascending=False).iloc[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'So when Pilate saw that he was gaining nothing, but rather that a riot was beginning, he took water and washed his hands before the crowd, saying, \"I am innocent of this man\\'s blood; see to it yourselves.\"'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "esv.text(reference.init('Matthew', 27, 24))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'PURIFICATION', 'HYPOCRISY', 'WASHING', 'HAND', 'COURT', 'INNOCENCY', 'BLOOD', 'PILATE, PONTIUS', 'HOMICIDE', 'OPINION, PUBLIC', 'GOVERNMENT', 'JESUS, THE CHRIST', 'RESPONSIBILITY', 'POLITICS', 'PRAYER', 'CAPERNAUM', 'DEMAGOGISM', 'BARABBAS', 'JUDGE', 'PRISONERS', 'COMPLICITY', 'VERDICT', 'ABLUTION', 'MONTH', 'RULERS'}\n"
     ]
    }
   ],
   "source": [
    "print(set([d['topic_name'] for d in nave.by_reference()['Matthew'][27][24]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>reference_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tpc:-2146356859</td>\n",
       "      <td>DANCING</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tpc:-2145736290</td>\n",
       "      <td>CORMORANT</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tpc:-2145523250</td>\n",
       "      <td>CANDACE</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tpc:-2145052645</td>\n",
       "      <td>CHRONOLOGY</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tpc:-2144306684</td>\n",
       "      <td>GINNETHON</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                id       label  reference_count\n",
       "0  tpc:-2146356859     DANCING               28\n",
       "1  tpc:-2145736290   CORMORANT                4\n",
       "2  tpc:-2145523250     CANDACE                1\n",
       "3  tpc:-2145052645  CHRONOLOGY                1\n",
       "4  tpc:-2144306684   GINNETHON                3"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.by_topic_nodes().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Passage(name='Leviticus 11:17', references=[Reference(book='Leviticus', chapter=11, verse=17)], text='the little owl, the cormorant, the short-eared owl,'),\n",
       " Passage(name='Deuteronomy 14:17', references=[Reference(book='Deuteronomy', chapter=14, verse=17)], text='and the tawny owl, the carrion vulture and the cormorant,'),\n",
       " Passage(name='Isaiah 34:11', references=[Reference(book='Isaiah', chapter=34, verse=11)], text='But the hawk and the porcupine shall possess it, the owl and the raven shall dwell in it. He shall stretch the line of confusion over it, and the plumb line of emptiness.'),\n",
       " Passage(name='Zephaniah 2:14', references=[Reference(book='Zephaniah', chapter=2, verse=14)], text='Herds shall lie down in her midst, all kinds of beasts; even the owl and the hedgehog shall lodge in her capitals; a voice shall hoot in the window; devastation will be on the threshold; for her cedar work will be laid bare.')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.by_topic()['CORMORANT']['A bird forbidden as food']['General reference(s) to this category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>reference_count</th>\n",
       "      <th>references</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tpc:-2145736290</td>\n",
       "      <td>CORMORANT</td>\n",
       "      <td>4</td>\n",
       "      <td>((Deuteronomy, 14, 17), (Isaiah, 34, 11), (Zep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>tpc:-2134662781</td>\n",
       "      <td>OWL</td>\n",
       "      <td>12</td>\n",
       "      <td>((Isaiah, 34, 13), (Leviticus, 11, 17), (Isaia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>tpc:-1883475154</td>\n",
       "      <td>FOOD</td>\n",
       "      <td>111</td>\n",
       "      <td>((1 Samuel, 25, 18), (Romans, 14, 14), (Leviti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736</th>\n",
       "      <td>tpc:-1469473955</td>\n",
       "      <td>SANITATION</td>\n",
       "      <td>545</td>\n",
       "      <td>((Deuteronomy, 14, 26), (Leviticus, 13, 39), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2286</th>\n",
       "      <td>tpc:-46757338</td>\n",
       "      <td>BIRDS</td>\n",
       "      <td>138</td>\n",
       "      <td>((Jeremiah, 5, 28), (Genesis, 1, 24), (Jeremia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3499</th>\n",
       "      <td>tpc:1093495174</td>\n",
       "      <td>UNCLEAN</td>\n",
       "      <td>76</td>\n",
       "      <td>((Deuteronomy, 14, 13), (Leviticus, 11, 26), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4141</th>\n",
       "      <td>tpc:1688371376</td>\n",
       "      <td>ANIMALS</td>\n",
       "      <td>648</td>\n",
       "      <td>((Exodus, 9, 5), (Numbers, 22, 24), (Job, 40, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id       label  reference_count  \\\n",
       "1     tpc:-2145736290   CORMORANT                4   \n",
       "17    tpc:-2134662781         OWL               12   \n",
       "295   tpc:-1883475154        FOOD              111   \n",
       "736   tpc:-1469473955  SANITATION              545   \n",
       "2286    tpc:-46757338       BIRDS              138   \n",
       "3499   tpc:1093495174     UNCLEAN               76   \n",
       "4141   tpc:1688371376     ANIMALS              648   \n",
       "\n",
       "                                             references  \n",
       "1     ((Deuteronomy, 14, 17), (Isaiah, 34, 11), (Zep...  \n",
       "17    ((Isaiah, 34, 13), (Leviticus, 11, 17), (Isaia...  \n",
       "295   ((1 Samuel, 25, 18), (Romans, 14, 14), (Leviti...  \n",
       "736   ((Deuteronomy, 14, 26), (Leviticus, 13, 39), (...  \n",
       "2286  ((Jeremiah, 5, 28), (Genesis, 1, 24), (Jeremia...  \n",
       "3499  ((Deuteronomy, 14, 13), (Leviticus, 11, 26), (...  \n",
       "4141  ((Exodus, 9, 5), (Numbers, 22, 24), (Job, 40, ...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ecce.passage as passage\n",
    "\n",
    "nave.topics_frame(passage.init([reference.init('Leviticus', 11, 17)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional Exploratory Analysis\n",
    "\n",
    "More exploratory data analysis is available from https://ecce.rcd.ai/eda.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['JESUS', 'THE CHRIST']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.extract_topics_of(dict(topic_name='JESUS, THE CHRIST'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x13337 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 8 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave_data.tokenize(['In the beginning, God created the heavens and the earth.'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_text (24515, 13337)\n",
      "train_topics (24515, 853)\n",
      "test_text (6129, 13337)\n",
      "test_topics (6129, 853)\n"
     ]
    }
   ],
   "source": [
    "train_text, test_text, train_topics, test_topics = nave_data.data_split()\n",
    "\n",
    "print('train_text', train_text.shape)\n",
    "print('train_topics', train_topics.shape)\n",
    "print('test_text', test_text.shape)\n",
    "print('test_topics', test_topics.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[TopicResult(probability=0.1233959, id='tpc:1494754856', label='QUOTATIONS AND ALLUSIONS'),\n",
       " TopicResult(probability=0.08785395, id='tpc:1487961918', label='JOB'),\n",
       " TopicResult(probability=0.071867436, id='tpc:-175446433', label='PSALMS'),\n",
       " TopicResult(probability=0.04190604, id='tpc:-817545140', label='MAN'),\n",
       " TopicResult(probability=0.04133786, id='tpc:-1808138860', label='PRAISE'),\n",
       " TopicResult(probability=0.032243002, id='tpc:-1481495806', label='PROPHECY'),\n",
       " TopicResult(probability=0.024686696, id='tpc:1455684257', label='CONDESCENSION OF GOD'),\n",
       " TopicResult(probability=0.02251567, id='tpc:-1195302428', label='SIN (1)'),\n",
       " TopicResult(probability=0.021504914, id='tpc:1324765133', label='RELIGION')]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ecce.model.nave.model import NaveModel\n",
    "\n",
    "nm = NaveModel()\n",
    "nm.load_weights('ecce/data/checkpoints/nave-4576e8.hdf5')\n",
    "nm.predict('What is man that you are mindful of him?', threshold=0.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Splitting train/val/test data...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating...\n",
      "6129/6129 [==============================] - 0s 74us/step\n",
      "Accuracy 13.61%\n"
     ]
    }
   ],
   "source": [
    "nm.evaluate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Result Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>reference_count</th>\n",
       "      <th>simple_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2780</th>\n",
       "      <td>tpc:438585048</td>\n",
       "      <td>SIMON</td>\n",
       "      <td>40</td>\n",
       "      <td>SIMON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4432</th>\n",
       "      <td>tpc:1955587128</td>\n",
       "      <td>PARSIMONY (STINGINESS)</td>\n",
       "      <td>13</td>\n",
       "      <td>PARSIMONY STINGINESS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4337</th>\n",
       "      <td>tpc:1868692075</td>\n",
       "      <td>SIMONY</td>\n",
       "      <td>2</td>\n",
       "      <td>SIMONY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                   label  reference_count  \\\n",
       "2780   tpc:438585048                   SIMON               40   \n",
       "4432  tpc:1955587128  PARSIMONY (STINGINESS)               13   \n",
       "4337  tpc:1868692075                  SIMONY                2   \n",
       "\n",
       "              simple_label  \n",
       "2780                 SIMON  \n",
       "4432  PARSIMONY STINGINESS  \n",
       "4337                SIMONY  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.topics_matching_extracted('SIMON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'tpc:-314322582',\n",
       " 'label': 'JESUS, THE CHRIST',\n",
       " 'reference_count': 10151,\n",
       " 'simple_label': 'JESUS, THE CHRIST'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nave.best_match_topic_for('JESUS')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Treasury of Scripture Knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ecce/data/tsk/tskxref.txt',\n",
       " 'ecce/data/tsk/parsed.csv',\n",
       " 'ecce/data/tsk/readme.txt']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glob('ecce/data/tsk/*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book</th>\n",
       "      <th>chapter</th>\n",
       "      <th>verse</th>\n",
       "      <th>sort_order</th>\n",
       "      <th>phrase</th>\n",
       "      <th>reference_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>beginning</td>\n",
       "      <td>pr 8:22-24;pr 16:4;mr 13:19;joh 1:1-3;heb 1:10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>God</td>\n",
       "      <td>ex 20:11;ex 31:18;1ch 16:26;ne 9:6;job 26:13;j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>without</td>\n",
       "      <td>job 26:7;isa 45:18;jer 4:23;na 2:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Spirit</td>\n",
       "      <td>job 26:14;ps 33:6;ps 104:30;isa 40:12-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>God</td>\n",
       "      <td>ps 33:6,9;ps 148:5;mt 8:3;joh 11:43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      book  chapter  verse  sort_order     phrase  \\\n",
       "0  Genesis        1      1           2  beginning   \n",
       "1  Genesis        1      1           3        God   \n",
       "2  Genesis        1      2           1    without   \n",
       "3  Genesis        1      2           2     Spirit   \n",
       "4  Genesis        1      3           1        God   \n",
       "\n",
       "                                      reference_list  \n",
       "0  pr 8:22-24;pr 16:4;mr 13:19;joh 1:1-3;heb 1:10...  \n",
       "1  ex 20:11;ex 31:18;1ch 16:26;ne 9:6;job 26:13;j...  \n",
       "2                job 26:7;isa 45:18;jer 4:23;na 2:10  \n",
       "3           job 26:14;ps 33:6;ps 104:30;isa 40:12-14  \n",
       "4                ps 33:6,9;ps 148:5;mt 8:3;joh 11:43  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ecce.tsk as tsk\n",
    "import ecce.model.tsk.data as tsk_data\n",
    "\n",
    "tsk.df().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pr 8:22-24;pr 16:4;mr 13:19;joh 1:1-3;heb 1:10;1jo 1:1'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsk.df().at[0, 'reference_list']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Reference(book='Proverbs', chapter=8, verse=22),\n",
       " Reference(book='Proverbs', chapter=8, verse=23),\n",
       " Reference(book='Proverbs', chapter=8, verse=24),\n",
       " Reference(book='Proverbs', chapter=16, verse=4),\n",
       " Reference(book='Mark', chapter=13, verse=19),\n",
       " Reference(book='John', chapter=1, verse=1),\n",
       " Reference(book='John', chapter=1, verse=2),\n",
       " Reference(book='John', chapter=1, verse=3),\n",
       " Reference(book='Hebrews', chapter=1, verse=10),\n",
       " Reference(book='1 John', chapter=1, verse=1)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsk.parse(tsk.df().at[0, 'reference_list'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>linked_book</th>\n",
       "      <th>linked_chapter</th>\n",
       "      <th>linked_verse</th>\n",
       "      <th>phrase</th>\n",
       "      <th>book</th>\n",
       "      <th>chapter</th>\n",
       "      <th>verse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>74</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>104</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Isaiah</td>\n",
       "      <td>45</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       uuid linked_book  linked_chapter  linked_verse    phrase     book  \\\n",
       "0  93a9c7b7     Genesis               1             5  Day, and  Genesis   \n",
       "1  93a9c7b7     Genesis               1             5  Day, and   Psalms   \n",
       "2  93a9c7b7     Genesis               1             5  Day, and   Psalms   \n",
       "3  93a9c7b7     Genesis               1             5  Day, and   Psalms   \n",
       "4  93a9c7b7     Genesis               1             5  Day, and   Isaiah   \n",
       "\n",
       "   chapter  verse  \n",
       "0        8     22  \n",
       "1       19      2  \n",
       "2       74     16  \n",
       "3      104     20  \n",
       "4       45      7  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsk.init().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>linked_book</th>\n",
       "      <th>linked_chapter</th>\n",
       "      <th>linked_verse</th>\n",
       "      <th>phrase</th>\n",
       "      <th>book</th>\n",
       "      <th>chapter</th>\n",
       "      <th>verse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>74</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Psalms</td>\n",
       "      <td>104</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Isaiah</td>\n",
       "      <td>45</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Jeremiah</td>\n",
       "      <td>33</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>1 Corinthians</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>Ephesians</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>93a9c7b7</td>\n",
       "      <td>Genesis</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Day, and</td>\n",
       "      <td>1 Thessalonians</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       uuid linked_book  linked_chapter  linked_verse    phrase  \\\n",
       "0  93a9c7b7     Genesis               1             5  Day, and   \n",
       "1  93a9c7b7     Genesis               1             5  Day, and   \n",
       "2  93a9c7b7     Genesis               1             5  Day, and   \n",
       "3  93a9c7b7     Genesis               1             5  Day, and   \n",
       "4  93a9c7b7     Genesis               1             5  Day, and   \n",
       "5  93a9c7b7     Genesis               1             5  Day, and   \n",
       "6  93a9c7b7     Genesis               1             5  Day, and   \n",
       "7  93a9c7b7     Genesis               1             5  Day, and   \n",
       "8  93a9c7b7     Genesis               1             5  Day, and   \n",
       "\n",
       "              book  chapter  verse  \n",
       "0          Genesis        8     22  \n",
       "1           Psalms       19      2  \n",
       "2           Psalms       74     16  \n",
       "3           Psalms      104     20  \n",
       "4           Isaiah       45      7  \n",
       "5         Jeremiah       33     20  \n",
       "6    1 Corinthians        3     13  \n",
       "7        Ephesians        5     13  \n",
       "8  1 Thessalonians        5      5  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsk.find_by_uuid('93a9c7b7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Passage(name='Genesis 8:22', references=[Reference(book='Genesis', chapter=8, verse=22)], text='While the earth remains, seedtime and harvest, cold and heat, summer and winter, day and night, shall not cease.\"')]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsk.passages_by_uuid('93a9c7b7', include_text=True)[0:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_text (24831, 150)\n",
      "train_clusters (24831, 63581)\n",
      "test_text (6208, 150)\n",
      "test_clusters (6208, 63581)\n"
     ]
    }
   ],
   "source": [
    "train_text, test_text, train_clusters, test_clusters = tsk_data.data_split()\n",
    "\n",
    "print('train_text', train_text.shape)\n",
    "print('train_clusters', train_clusters.shape)\n",
    "print('test_text', test_text.shape)\n",
    "print('test_clusters', test_clusters.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "INFO:root:Loading ESV JSON...\n"
     ]
    }
   ],
   "source": [
    "from ecce.model.tsk.model import ClusterModel\n",
    "\n",
    "cm = ClusterModel()\n",
    "cm.load_weights('ecce/data/checkpoints/tsk-cluster-8a1db9.hdf5')\n",
    "result = cm.predict('What is man that you are mindful of him?', n_max=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.018426116555929184,\n",
      "  '00b37c33',\n",
      "  ['Isaiah', 2, 22],\n",
      "  'Stop regarding man in whose nostrils is breath, for of what account is he?',\n",
      "  [['Job 7:15-21',\n",
      "    [['Job', 7, 15],\n",
      "     ['Job', 7, 16],\n",
      "     ['Job', 7, 17],\n",
      "     ['Job', 7, 18],\n",
      "     ['Job', 7, 19],\n",
      "     ['Job', 7, 20],\n",
      "     ['Job', 7, 21]],\n",
      "    '15 so that I would choose strangling and death rather than my bones.\\n'\n",
      "    '16 I loathe my life; I would not live forever. Leave me alone, for my '\n",
      "    'days are a breath.\\n'\n",
      "    '17 What is man, that you make so much of him, and that you set your heart '\n",
      "    'on him,\\n'\n",
      "    '18 visit him every morning and test him every moment?\\n'\n",
      "    '19 How long will you not look away from me, nor leave me alone till I '\n",
      "    'swallow my spit?\\n'\n",
      "    '20 If I sin, what do I do to you, you watcher of mankind? Why have you '\n",
      "    'made me your mark? Why have I become a burden to you?\\n'\n",
      "    '21 Why do you not pardon my transgression and take away my iniquity? For '\n",
      "    'now I shall lie in the earth; you will seek me, but I shall not be.\"'],\n",
      "   ['Psalms 8:4',\n",
      "    [['Psalms', 8, 4]],\n",
      "    'what is man that you are mindful of him, and the son of man that you care '\n",
      "    'for him?'],\n",
      "   ['Psalms 144:3-4',\n",
      "    [['Psalms', 144, 3], ['Psalms', 144, 4]],\n",
      "    '3 O LORD, what is man that you regard him, or the son of man that you '\n",
      "    'think of him?\\n'\n",
      "    '4 Man is like a breath; his days are like a passing shadow.']]],\n",
      " [0.01758749410510063,\n",
      "  '82022438',\n",
      "  ['Hebrews', 2, 6],\n",
      "  'It has been testified somewhere, \"What is man, that you are mindful of him, '\n",
      "  'or the son of man, that you care for him?',\n",
      "  [['Job 7:17-18',\n",
      "    [['Job', 7, 17], ['Job', 7, 18]],\n",
      "    '17 What is man, that you make so much of him, and that you set your heart '\n",
      "    'on him,\\n'\n",
      "    '18 visit him every morning and test him every moment?'],\n",
      "   ['Job 15:14',\n",
      "    [['Job', 15, 14]],\n",
      "    'What is man, that he can be pure? Or he who is born of a woman, that he '\n",
      "    'can be righteous?'],\n",
      "   ['Psalms 8:4-8',\n",
      "    [['Psalms', 8, 4],\n",
      "     ['Psalms', 8, 5],\n",
      "     ['Psalms', 8, 6],\n",
      "     ['Psalms', 8, 7],\n",
      "     ['Psalms', 8, 8]],\n",
      "    '4 what is man that you are mindful of him, and the son of man that you '\n",
      "    'care for him?\\n'\n",
      "    '5 Yet you have made him a little lower than the heavenly beings and '\n",
      "    'crowned him with glory and honor.\\n'\n",
      "    '6 You have given him dominion over the works of your hands; you have put '\n",
      "    'all things under his feet,\\n'\n",
      "    '7 all sheep and oxen, and also the beasts of the field,\\n'\n",
      "    '8 the birds of the heavens, and the fish of the sea, whatever passes '\n",
      "    'along the paths of the seas.'],\n",
      "   ['Psalms 144:3',\n",
      "    [['Psalms', 144, 3]],\n",
      "    'O LORD, what is man that you regard him, or the son of man that you think '\n",
      "    'of him?'],\n",
      "   ['Isaiah 40:17',\n",
      "    [['Isaiah', 40, 17]],\n",
      "    'All the nations are as nothing before him, they are accounted by him as '\n",
      "    'less than nothing and emptiness.']]]]\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "import json\n",
    "\n",
    "print_raw = compose(pprint, json.loads, json.dumps)\n",
    "\n",
    "print_raw(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ecce Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Skipping 73 verses with no assigned topics.\n"
     ]
    }
   ],
   "source": [
    "from ecce.model.ecce import EcceModel\n",
    "\n",
    "m = EcceModel('ecce/data/checkpoints/nave-4576e8.hdf5',\n",
    "              'ecce/data/checkpoints/tsk-cluster-8a1db9.hdf5')\n",
    "\n",
    "result = m.predict('What is man that you are mindful of him?', top_clusters=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.1233958974480629, 'tpc:1494754856', 'QUOTATIONS AND ALLUSIONS'],\n",
      " [0.08785395324230194, 'tpc:1487961918', 'JOB'],\n",
      " [0.071867436170578, 'tpc:-175446433', 'PSALMS'],\n",
      " [0.04190604016184807, 'tpc:-817545140', 'MAN'],\n",
      " [0.04133785888552666, 'tpc:-1808138860', 'PRAISE'],\n",
      " [0.03224300220608711, 'tpc:-1481495806', 'PROPHECY'],\n",
      " [0.024686696007847786, 'tpc:1455684257', 'CONDESCENSION OF GOD'],\n",
      " [0.022515669465065002, 'tpc:-1195302428', 'SIN (1)'],\n",
      " [0.02150491438806057, 'tpc:1324765133', 'RELIGION']]\n"
     ]
    }
   ],
   "source": [
    "print_raw(result.topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.018426116555929184,\n",
      "  '00b37c33',\n",
      "  ['Isaiah', 2, 22],\n",
      "  'Stop regarding man in whose nostrils is breath, for of what account is he?',\n",
      "  [['Job 7:15-21',\n",
      "    [['Job', 7, 15],\n",
      "     ['Job', 7, 16],\n",
      "     ['Job', 7, 17],\n",
      "     ['Job', 7, 18],\n",
      "     ['Job', 7, 19],\n",
      "     ['Job', 7, 20],\n",
      "     ['Job', 7, 21]],\n",
      "    '15 so that I would choose strangling and death rather than my bones.\\n'\n",
      "    '16 I loathe my life; I would not live forever. Leave me alone, for my '\n",
      "    'days are a breath.\\n'\n",
      "    '17 What is man, that you make so much of him, and that you set your heart '\n",
      "    'on him,\\n'\n",
      "    '18 visit him every morning and test him every moment?\\n'\n",
      "    '19 How long will you not look away from me, nor leave me alone till I '\n",
      "    'swallow my spit?\\n'\n",
      "    '20 If I sin, what do I do to you, you watcher of mankind? Why have you '\n",
      "    'made me your mark? Why have I become a burden to you?\\n'\n",
      "    '21 Why do you not pardon my transgression and take away my iniquity? For '\n",
      "    'now I shall lie in the earth; you will seek me, but I shall not be.\"'],\n",
      "   ['Psalms 8:4',\n",
      "    [['Psalms', 8, 4]],\n",
      "    'what is man that you are mindful of him, and the son of man that you care '\n",
      "    'for him?'],\n",
      "   ['Psalms 144:3-4',\n",
      "    [['Psalms', 144, 3], ['Psalms', 144, 4]],\n",
      "    '3 O LORD, what is man that you regard him, or the son of man that you '\n",
      "    'think of him?\\n'\n",
      "    '4 Man is like a breath; his days are like a passing shadow.']]],\n",
      " [0.01758749410510063,\n",
      "  '2d9b08f4',\n",
      "  ['Job', 9, 32],\n",
      "  'For he is not a man, as I am, that I might answer him, that we should come '\n",
      "  'to trial together.',\n",
      "  [['Numbers 23:19',\n",
      "    [['Numbers', 23, 19]],\n",
      "    'God is not man, that he should lie, or a son of man, that he should '\n",
      "    'change his mind. Has he said, and will he not do it? Or has he spoken, '\n",
      "    'and will he not fulfill it?'],\n",
      "   ['1 Samuel 16:7',\n",
      "    [['1 Samuel', 16, 7]],\n",
      "    'But the LORD said to Samuel, \"Do not look on his appearance or on the '\n",
      "    'height of his stature, because I have rejected him. For the LORD sees not '\n",
      "    'as man sees: man looks on the outward appearance, but the LORD looks on '\n",
      "    'the heart.\"'],\n",
      "   ['Job 33:12',\n",
      "    [['Job', 33, 12]],\n",
      "    '\"Behold, in this you are not right. I will answer you, for God is greater '\n",
      "    'than man.'],\n",
      "   ['Job 35:5-7',\n",
      "    [['Job', 35, 5], ['Job', 35, 6], ['Job', 35, 7]],\n",
      "    '5 Look at the heavens, and see; and behold the clouds, which are higher '\n",
      "    'than you.\\n'\n",
      "    '6 If you have sinned, what do you accomplish against him? And if your '\n",
      "    'transgressions are multiplied, what do you do to him?\\n'\n",
      "    '7 If you are righteous, what do you give to him? Or what does he receive '\n",
      "    'from your hand?'],\n",
      "   ['Ecclesiastes 6:10',\n",
      "    [['Ecclesiastes', 6, 10]],\n",
      "    'Whatever has come to be has already been named, and it is known what man '\n",
      "    'is, and that he is not able to dispute with one stronger than he.'],\n",
      "   ['Isaiah 45:9',\n",
      "    [['Isaiah', 45, 9]],\n",
      "    '\"Woe to him who strives with him who formed him, a pot among earthen '\n",
      "    \"pots! Does the clay say to him who forms it, 'What are you making?' or \"\n",
      "    \"'Your work has no handles'?\"],\n",
      "   ['Jeremiah 49:19',\n",
      "    [['Jeremiah', 49, 19]],\n",
      "    'Behold, like a lion coming up from the jungle of the Jordan against a '\n",
      "    'perennial pasture, I will suddenly make him run away from her. And I will '\n",
      "    'appoint over her whomever I choose. For who is like me? Who will summon '\n",
      "    'me? What shepherd can stand before me?'],\n",
      "   ['Romans 9:20',\n",
      "    [['Romans', 9, 20]],\n",
      "    'But who are you, O man, to answer back to God? Will what is molded say to '\n",
      "    'its molder, \"Why have you made me like this?\"'],\n",
      "   ['1 John 3:20',\n",
      "    [['1 John', 3, 20]],\n",
      "    'for whenever our heart condemns us, God is greater than our heart, and he '\n",
      "    'knows everything.']]]]\n"
     ]
    }
   ],
   "source": [
    "print_raw(result.clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.12403968987829263, 'tpc:-1776404894', 'PHILOSOPHY'],\n",
      " [0.12403968987829263, 'tpc:1071709628', 'ELIHU'],\n",
      " [0.12403968987829263, 'tpc:1528792171', 'INFIDELITY'],\n",
      " [0.07421056689121147, 'tpc:-1561663083', 'MOABITES'],\n",
      " [0.07421056689121147, 'tpc:-1347520606', 'PISGAH'],\n",
      " [0.07421056689121147, 'tpc:916989502', 'TRUTH'],\n",
      " [0.07421056689121147, 'tpc:1180418977', 'WORLDLINESS'],\n",
      " [0.07421056689121147, 'tpc:1860581920', 'SORCERY'],\n",
      " [0.07421056689121147, 'tpc:-2015850403', 'SECRET'],\n",
      " [0.07421056689121147, 'tpc:-17202098', 'CHURCH AND STATE']]\n"
     ]
    }
   ],
   "source": [
    "print_raw(result.passage_topics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}