experiments/python_parsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Experiments in splitting Python code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "import astor\n",
    "import tokenize\n",
    "import io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(\"experiments/fixtures/forest-fire.py\", \"r\")\n",
    "code = file.read()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def\ninitialise\n(\n)\n:\ngrid\n=\n{\n(\nx\n,\ny\n)\n:\n(\ntree\nif\nrandom\n.\nrandom\n(\n)\n<=\ninitial_trees\nelse\nspace\n)\nfor\nx\nin\nrange\n(\nL\n)\nfor\ny\nin\nrange\n(\nL\n)\n}\nreturn\ngrid\ndef\ngprint\n(\ngrid\n)\n:\ntxt\n=\n.\njoin\n(\n.\njoin\n(\ngrid\n[\nx\n,\ny\n]\nfor\nx\nin\nrange\n(\nL\n)\n)\nfor\ny\nin\nrange\n(\nL\n)\n)\nprint\n(\ntxt\n)\ndef\nquickprint\n(\ngrid\n)\n:\nt\n=\nb\n=\nll\n=\nL\n*\nL\nfor\nx\nin\nrange\n(\nL\n)\n:\nfor\ny\nin\nrange\n(\nL\n)\n:\nif\ngrid\n[\nx\n,\ny\n]\nin\n(\ntree\n,\nburning\n)\n:\nt\n+=\nif\ngrid\n[\nx\n,\ny\n]\n==\nburning\n:\nb\n+=\nprint\n(\n(\n+\n)\n%\n(\nll\n,\nt\n,\nb\n,\n*\nt\n/\nll\n,\n*\nb\n/\nll\n)\n)\ndef\ngnew\n(\ngrid\n)\n:\nnewgrid\n=\n{\n}\nfor\nx\nin\nrange\n(\nL\n)\n:\nfor\ny\nin\nrange\n(\nL\n)\n:\nif\ngrid\n[\nx\n,\ny\n]\n==\nburning\n:\nnewgrid\n[\nx\n,\ny\n]\n=\nspace\nelif\ngrid\n[\nx\n,\ny\n]\n==\nspace\n:\nnewgrid\n[\nx\n,\ny\n]\n=\ntree\nif\nrandom\n.\nrandom\n(\n)\n<=\np\nelse\nspace\nelif\ngrid\n[\nx\n,\ny\n]\n==\ntree\n:\nnewgrid\n[\nx\n,\ny\n]\n=\nburning\nif\nany\n(\ngrid\n.\nget\n(\n(\nx\n+\ndx\n,\ny\n+\ndy\n)\n,\nspace\n)\n==\nburning\nfor\ndx\n,\ndy\nin\nhood\n)\nor\nrandom\n.\nrandom\n(\n)\n<=\nf\nelse\ntree\nreturn\nnewgrid\n"
     ]
    }
   ],
   "source": [
    "tree = ast.parse(code)\n",
    "funcs = []\n",
    "for node in ast.walk(tree):\n",
    "    if isinstance(node, ast.FunctionDef):\n",
    "        funcs.append(astor.to_source(node))\n",
    "\n",
    "for func in funcs: \n",
    "    tokens = tokenize.generate_tokens(io.StringIO(func).readline)\n",
    "    tokens = [token for t in tokens if t.type == tokenize.NAME or t.type == tokenize.OP for token in t.string.split(\" \")]\n",
    "    for token in tokens:\n",
    "        print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'_io.StringIO' object is not callable",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-86-2af5f4182340>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \"\"\"\n\u001b[1;32m     10\u001b[0m \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenize\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_tokens\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStringIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfrag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/tokenize.py\u001b[0m in \u001b[0;36m_tokenize\u001b[0;34m(readline, encoding)\u001b[0m\n\u001b[1;32m    510\u001b[0m     \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m             \u001b[0;31m# loop over lines in stream\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    511\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 512\u001b[0;31m             \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    513\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    514\u001b[0m             \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mb''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: '_io.StringIO' object is not callable"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}