programs/formats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Formats ins and outs\n",
    "\n",
    "See also the [docs](https://annotation.github.io/text-fabric/Api/Text/#text-representation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from tf.fabric import Fabric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is Text-Fabric 7.6.8\n",
      "Api reference : https://annotation.github.io/text-fabric/Api/Fabric/\n",
      "\n",
      "10 features found and 0 ignored\n"
     ]
    }
   ],
   "source": [
    "GH_BASE = os.path.expanduser('~/github')\n",
    "ORG = 'annotation'\n",
    "REPO = 'banks'\n",
    "FOLDER = 'tf'\n",
    "TF_DIR = f'{GH_BASE}/{ORG}/{REPO}/{FOLDER}'\n",
    "\n",
    "VERSION = '0.2'\n",
    "\n",
    "TF_PATH = f'{TF_DIR}/{VERSION}'\n",
    "TF = Fabric(locations=TF_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We ask for a list of all features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('author',\n",
       " 'gap',\n",
       " 'letters',\n",
       " 'number',\n",
       " 'otype',\n",
       " 'punc',\n",
       " 'terminator',\n",
       " 'title',\n",
       " 'oslots')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allFeatures = TF.explore(silent=True, show=True)\n",
    "loadableFeatures = allFeatures['nodes'] + allFeatures['edges']\n",
    "loadableFeatures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We load all features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.00s loading features ...\n",
      "   |     0.00s B otype                from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B oslots               from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B title                from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B number               from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B letters              from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B punc                 from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B terminator           from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B author               from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "   |     0.00s B gap                  from /Users/dirk/github/annotation/banks/tf/0.2\n",
      "  0.03s All features loaded/computed - for details use loadLog()\n"
     ]
    }
   ],
   "source": [
    "api = TF.load(loadableFeatures, silent=False)\n",
    "T = api.T\n",
    "F = api.F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'line-default': 'word', 'line-term': 'line', 'text-orig-full': 'word'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = F.otype.s('word')\n",
    "lines = F.otype.s('line')\n",
    "sents = F.otype.s('sentence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "explain = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# single line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : implicit\n",
      "\tdescend: implicit\n",
      "\tfunc   : no  custom format implementation\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (format target type)\n",
      "\t\tEXPANSION: 1 line 103\n",
      "\t\tFORMATTING: implicit line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \"XXX, \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'XXX, '"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0], explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : implicit\n",
      "\tdescend: True\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: word  (descend=True) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 1, 2, 3\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      " 1m 43s \t\tMATERIAL:\n",
      "\t\t\tword 1 ADDS \"Everything \"\n",
      "\t\t\tword 2 ADDS \"about \"\n",
      "\t\t\tword 3 ADDS \"us \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Everything about us '"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0], descend=True, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : line-term targeted at line\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: line 103\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "    56s \t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \", \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "', '"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0], fmt='line-term', explain=explain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# two lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (node type)\n",
      "\t\tEXPANSION: line 103\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \"XXX, \"\n",
      "\tNODE: line 104\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (node type)\n",
      "\t\tEXPANSION: line 104\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 104 ADDS \"XXX, \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'XXX, XXX, '"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0:2], explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: True\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: word  (descend=True) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 1, 2, 3\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 1 ADDS \"Everything \"\n",
      "\t\t\tword 2 ADDS \"about \"\n",
      "\t\t\tword 3 ADDS \"us \"\n",
      "\tNODE: line 104\n",
      "\t\tTARGET LEVEL: word  (descend=True) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 4, 5, 6\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 4 ADDS \"everything \"\n",
      "\t\t\tword 5 ADDS \"around \"\n",
      "\t\t\tword 6 ADDS \"us \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Everything about us everything around us '"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0:2], descend=True, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : line-term targeted at line\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: line 103\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: line 103\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \", \"\n",
      "\tNODE: line 104\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: line 104\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 104 ADDS \", \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "', , '"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(lines[0:2], fmt='line-term', explain=explain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# single sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : implicit\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: word  (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 1 ADDS \"Everything \"\n",
      "\t\t\tword 2 ADDS \"about \"\n",
      "\t\t\tword 3 ADDS \"us, \"\n",
      "\t\t\tword 4 ADDS \"everything \"\n",
      "\t\t\tword 5 ADDS \"around \"\n",
      "\t\t\tword 6 ADDS \"us, \"\n",
      "\t\t\tword 7 ADDS \"everything \"\n",
      "\t\t\tword 8 ADDS \"we \"\n",
      "\t\t\tword 9 ADDS \"know \"\n",
      "\t\t\tword 10 ADDS \"and \"\n",
      "\t\t\tword 11 ADDS \"can \"\n",
      "\t\t\tword 12 ADDS \"know \"\n",
      "\t\t\tword 13 ADDS \"of \"\n",
      "\t\t\tword 14 ADDS \"is \"\n",
      "\t\t\tword 15 ADDS \"composed \"\n",
      "\t\t\tword 16 ADDS \"ultimately \"\n",
      "\t\t\tword 17 ADDS \"of \"\n",
      "\t\t\tword 18 ADDS \"patterns \"\n",
      "\t\t\tword 19 ADDS \"of \"\n",
      "\t\t\tword 20 ADDS \"nothing; \"\n",
      "\t\t\tword 21 ADDS \"that’s \"\n",
      "\t\t\tword 22 ADDS \"the \"\n",
      "\t\t\tword 23 ADDS \"bottom \"\n",
      "\t\t\tword 24 ADDS \"line, \"\n",
      "\t\t\tword 25 ADDS \"the \"\n",
      "\t\t\tword 26 ADDS \"final \"\n",
      "\t\t\tword 27 ADDS \"truth. \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Everything about us, everything around us, everything we know and can know of is composed ultimately of patterns of nothing; that’s the bottom line, the final truth. '"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0], explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : implicit\n",
      "\tdescend: False\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: sentence (descend=False) (no expansion)\n",
      "\t\tEXPANSION: sentence 115\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tsentence 115 ADDS \" \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' '"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0], descend=False, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : single node\n",
      "\tfmt    : line-term targeted at line\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines 103, 104, 105, 106\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \", \"\n",
      "\t\t\tline 104 ADDS \", \"\n",
      "\t\t\tline 105 ADDS \"; \"\n",
      "\t\t\tline 106 ADDS \". \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "', , ; . '"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0], fmt='line-term', explain=explain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# two sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: word  (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 1 ADDS \"Everything \"\n",
      "\t\t\tword 2 ADDS \"about \"\n",
      "\t\t\tword 3 ADDS \"us, \"\n",
      "\t\t\tword 4 ADDS \"everything \"\n",
      "\t\t\tword 5 ADDS \"around \"\n",
      "\t\t\tword 6 ADDS \"us, \"\n",
      "\t\t\tword 7 ADDS \"everything \"\n",
      "\t\t\tword 8 ADDS \"we \"\n",
      "\t\t\tword 9 ADDS \"know \"\n",
      "\t\t\tword 10 ADDS \"and \"\n",
      "\t\t\tword 11 ADDS \"can \"\n",
      "\t\t\tword 12 ADDS \"know \"\n",
      "\t\t\tword 13 ADDS \"of \"\n",
      "\t\t\tword 14 ADDS \"is \"\n",
      "\t\t\tword 15 ADDS \"composed \"\n",
      "\t\t\tword 16 ADDS \"ultimately \"\n",
      "\t\t\tword 17 ADDS \"of \"\n",
      "\t\t\tword 18 ADDS \"patterns \"\n",
      "\t\t\tword 19 ADDS \"of \"\n",
      "\t\t\tword 20 ADDS \"nothing; \"\n",
      "\t\t\tword 21 ADDS \"that’s \"\n",
      "\t\t\tword 22 ADDS \"the \"\n",
      "\t\t\tword 23 ADDS \"bottom \"\n",
      "\t\t\tword 24 ADDS \"line, \"\n",
      "\t\t\tword 25 ADDS \"the \"\n",
      "\t\t\tword 26 ADDS \"final \"\n",
      "\t\t\tword 27 ADDS \"truth. \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: word  (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 28 ADDS \"So \"\n",
      "\t\t\tword 29 ADDS \"where \"\n",
      "\t\t\tword 30 ADDS \"we \"\n",
      "\t\t\tword 31 ADDS \"find \"\n",
      "\t\t\tword 32 ADDS \"we \"\n",
      "\t\t\tword 33 ADDS \"have \"\n",
      "\t\t\tword 34 ADDS \"any \"\n",
      "\t\t\tword 35 ADDS \"control \"\n",
      "\t\t\tword 36 ADDS \"over \"\n",
      "\t\t\tword 37 ADDS \"those \"\n",
      "\t\t\tword 38 ADDS \"patterns, \"\n",
      "\t\t\tword 39 ADDS \"why \"\n",
      "\t\t\tword 40 ADDS \"not \"\n",
      "\t\t\tword 41 ADDS \"make \"\n",
      "\t\t\tword 42 ADDS \"the \"\n",
      "\t\t\tword 43 ADDS \"most \"\n",
      "\t\t\tword 44 ADDS \"elegant \"\n",
      "\t\t\tword 45 ADDS \"ones, \"\n",
      "\t\t\tword 46 ADDS \"the \"\n",
      "\t\t\tword 47 ADDS \"most \"\n",
      "\t\t\tword 48 ADDS \"enjoyable \"\n",
      "\t\t\tword 49 ADDS \"and \"\n",
      "\t\t\tword 50 ADDS \"good \"\n",
      "\t\t\tword 51 ADDS \"ones, \"\n",
      "\t\t\tword 52 ADDS \"in \"\n",
      "\t\t\tword 53 ADDS \"our \"\n",
      "\t\t\tword 54 ADDS \"own \"\n",
      "\t\t\tword 55 ADDS \"terms? \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Everything about us, everything around us, everything we know and can know of is composed ultimately of patterns of nothing; that’s the bottom line, the final truth. So where we find we have any control over those patterns, why not make the most elegant ones, the most enjoyable and good ones, in our own terms? '"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0:2], explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: False\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: sentence (descend=False) (no expansion)\n",
      "\t\tEXPANSION: sentence 115\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tsentence 115 ADDS \" \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: sentence (descend=False) (no expansion)\n",
      "\t\tEXPANSION: sentence 116\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tsentence 116 ADDS \" \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'  '"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0:2], descend=False, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 2 nodes\n",
      "\tfmt    : line-term targeted at line\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines 103, 104, 105, 106\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \", \"\n",
      "\t\t\tline 104 ADDS \", \"\n",
      "\t\t\tline 105 ADDS \"; \"\n",
      "\t\t\tline 106 ADDS \". \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines 107, 108, 109\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 107 ADDS \", \"\n",
      "\t\t\tline 108 ADDS \", \"\n",
      "\t\t\tline 109 ADDS \"? \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "', , ; . , , ? '"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(sents[0:2], fmt='line-term', explain=explain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mixed content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = list(words[50:53]) + list(lines[4:6]) + list(sents[0:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 7 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: word 51\n",
      "\t\tTARGET LEVEL: word (no expansion needed) (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: word 51\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 51 ADDS \"ones, \"\n",
      "\tNODE: word 52\n",
      "\t\tTARGET LEVEL: word (no expansion needed) (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: word 52\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 52 ADDS \"in \"\n",
      "\tNODE: word 53\n",
      "\t\tTARGET LEVEL: word (no expansion needed) (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: word 53\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 53 ADDS \"our \"\n",
      "\tNODE: line 107\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (node type)\n",
      "\t\tEXPANSION: line 107\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 107 ADDS \"XXX, \"\n",
      "\tNODE: line 108\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (node type)\n",
      "\t\tEXPANSION: line 108\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 108 ADDS \"XXX, \"\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: word  (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 1 ADDS \"Everything \"\n",
      "\t\t\tword 2 ADDS \"about \"\n",
      "\t\t\tword 3 ADDS \"us, \"\n",
      "\t\t\tword 4 ADDS \"everything \"\n",
      "\t\t\tword 5 ADDS \"around \"\n",
      "\t\t\tword 6 ADDS \"us, \"\n",
      "\t\t\tword 7 ADDS \"everything \"\n",
      "\t\t\tword 8 ADDS \"we \"\n",
      "\t\t\tword 9 ADDS \"know \"\n",
      "\t\t\tword 10 ADDS \"and \"\n",
      "\t\t\tword 11 ADDS \"can \"\n",
      "\t\t\tword 12 ADDS \"know \"\n",
      "\t\t\tword 13 ADDS \"of \"\n",
      "\t\t\tword 14 ADDS \"is \"\n",
      "\t\t\tword 15 ADDS \"composed \"\n",
      "\t\t\tword 16 ADDS \"ultimately \"\n",
      "\t\t\tword 17 ADDS \"of \"\n",
      "\t\t\tword 18 ADDS \"patterns \"\n",
      "\t\t\tword 19 ADDS \"of \"\n",
      "\t\t\tword 20 ADDS \"nothing; \"\n",
      "\t\t\tword 21 ADDS \"that’s \"\n",
      "\t\t\tword 22 ADDS \"the \"\n",
      "\t\t\tword 23 ADDS \"bottom \"\n",
      "\t\t\tword 24 ADDS \"line, \"\n",
      "\t\t\tword 25 ADDS \"the \"\n",
      "\t\t\tword 26 ADDS \"final \"\n",
      "\t\t\tword 27 ADDS \"truth. \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: word  (descend=None) (target of implicit text-orig-full)\n",
      "\t\tEXPANSION: words 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 28 ADDS \"So \"\n",
      "\t\t\tword 29 ADDS \"where \"\n",
      "\t\t\tword 30 ADDS \"we \"\n",
      "\t\t\tword 31 ADDS \"find \"\n",
      "\t\t\tword 32 ADDS \"we \"\n",
      "\t\t\tword 33 ADDS \"have \"\n",
      "\t\t\tword 34 ADDS \"any \"\n",
      "\t\t\tword 35 ADDS \"control \"\n",
      "\t\t\tword 36 ADDS \"over \"\n",
      "\t\t\tword 37 ADDS \"those \"\n",
      "\t\t\tword 38 ADDS \"patterns, \"\n",
      "\t\t\tword 39 ADDS \"why \"\n",
      "\t\t\tword 40 ADDS \"not \"\n",
      "\t\t\tword 41 ADDS \"make \"\n",
      "\t\t\tword 42 ADDS \"the \"\n",
      "\t\t\tword 43 ADDS \"most \"\n",
      "\t\t\tword 44 ADDS \"elegant \"\n",
      "\t\t\tword 45 ADDS \"ones, \"\n",
      "\t\t\tword 46 ADDS \"the \"\n",
      "\t\t\tword 47 ADDS \"most \"\n",
      "\t\t\tword 48 ADDS \"enjoyable \"\n",
      "\t\t\tword 49 ADDS \"and \"\n",
      "\t\t\tword 50 ADDS \"good \"\n",
      "\t\t\tword 51 ADDS \"ones, \"\n",
      "\t\t\tword 52 ADDS \"in \"\n",
      "\t\t\tword 53 ADDS \"our \"\n",
      "\t\t\tword 54 ADDS \"own \"\n",
      "\t\t\tword 55 ADDS \"terms? \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'ones, in our XXX, XXX, Everything about us, everything around us, everything we know and can know of is composed ultimately of patterns of nothing; that’s the bottom line, the final truth. So where we find we have any control over those patterns, why not make the most elegant ones, the most enjoyable and good ones, in our own terms? '"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(content, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 7 nodes\n",
      "\tfmt    : implicit\n",
      "\tdescend: False\n",
      "\n",
      "\tNODE: word 51\n",
      "\t\tTARGET LEVEL: word (descend=False) (no expansion)\n",
      "\t\tEXPANSION: word 51\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 51 ADDS \"ones, \"\n",
      "\tNODE: word 52\n",
      "\t\tTARGET LEVEL: word (descend=False) (no expansion)\n",
      "\t\tEXPANSION: word 52\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 52 ADDS \"in \"\n",
      "\tNODE: word 53\n",
      "\t\tTARGET LEVEL: word (descend=False) (no expansion)\n",
      "\t\tEXPANSION: word 53\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tword 53 ADDS \"our \"\n",
      "\tNODE: line 107\n",
      "\t\tTARGET LEVEL: line (descend=False) (no expansion)\n",
      "\t\tEXPANSION: line 107\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 107 ADDS \"XXX, \"\n",
      "\tNODE: line 108\n",
      "\t\tTARGET LEVEL: line (descend=False) (no expansion)\n",
      "\t\tEXPANSION: line 108\n",
      "\t\tFORMATTING: with node-specific default line-default\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 108 ADDS \"XXX, \"\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: sentence (descend=False) (no expansion)\n",
      "\t\tEXPANSION: sentence 115\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tsentence 115 ADDS \" \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: sentence (descend=False) (no expansion)\n",
      "\t\tEXPANSION: sentence 116\n",
      "\t\tFORMATTING: with overall default text-orig-full\n",
      "\t\tMATERIAL:\n",
      "\t\t\tsentence 116 ADDS \" \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'ones, in our XXX, XXX,   '"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(content, descend=False, explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "EXPLANATION: T.text) called with parameters:\n",
      "\tnodes  : iterable of 7 nodes\n",
      "\tfmt    : line-term targeted at line\n",
      "\tdescend: implicit\n",
      "\n",
      "\tNODE: word 51\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines \n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\tNODE: word 52\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines \n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\tNODE: word 53\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines \n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\tNODE: line 107\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: line 107\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 107 ADDS \", \"\n",
      "\tNODE: line 108\n",
      "\t\tTARGET LEVEL: line (no expansion needed) (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: line 108\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 108 ADDS \", \"\n",
      "\tNODE: sentence 115\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines 103, 104, 105, 106\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 103 ADDS \", \"\n",
      "\t\t\tline 104 ADDS \", \"\n",
      "\t\t\tline 105 ADDS \"; \"\n",
      "\t\t\tline 106 ADDS \". \"\n",
      "\tNODE: sentence 116\n",
      "\t\tTARGET LEVEL: line  (descend=None) (target of explicit line-term)\n",
      "\t\tEXPANSION: lines 107, 108, 109\n",
      "\t\tFORMATTING: with explicit line-term\n",
      "\t\tMATERIAL:\n",
      "\t\t\tline 107 ADDS \", \"\n",
      "\t\t\tline 108 ADDS \", \"\n",
      "\t\t\tline 109 ADDS \"? \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "', , , , ; . , , ? '"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.text(content, fmt='line-term', explain=explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "  var = locals()\n",
    "  def verbose(x):\n",
    "    exec(x, var)\n",
    "    \n",
    "  verbose('x = \"aap\"')\n",
    "  verbose('print(x)')\n",
    "  verbose('print(\"noot\")')\n",
    "  return 'OK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aap\n",
      "noot\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'OK'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}