record_dtype_experiments.ipynb

{
 "metadata": {
  "name": "record_dtype_experiments"
 }, 
 "nbformat": 2, 
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "import numpy as np", 
      "import scipy.weave as weave", 
      "", 
      "from IPython.core.display import publish_png", 
      "", 
      "%gui qt"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 1
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "# using record dtypes for state arrays", 
      "", 
      "The purpose of this branch is two-fold:", 
      "", 
      "1. we want to declare and access cells as being made up of multiple parts", 
      "2. we want those multiple parts to be displayable in a useful manner:", 
      "   1. interpret all values together as a number and use that for some palette", 
      "   2. interpret the parts as different parts and display them as such", 
      "", 
      "Additional support must be in place for exporting/importing/generating configs, but that's a lower priority.", 
      "", 
      "## a cell is more than one piece of information", 
      "", 
      "this bit is given to us for free by numpys record dtypes. here are a few examples"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "print \"this is how you could model a beta-asynchronous single cell.\"", 
      "a = np.zeros((4, 3), dtype=[(\"hidden\", \"int8\"), (\"communicated\", \"int8\")])", 
      "a[0][0][\"hidden\"] = 99", 
      "a[0][0][\"communicated\"] = 100", 
      "print a, a.dtype"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "this is how you could model a beta-asynchronous single cell.", 
        "[[(99, 100) (0, 0) (0, 0)]", 
        " [(0, 0) (0, 0) (0, 0)]", 
        " [(0, 0) (0, 0) (0, 0)]", 
        " [(0, 0) (0, 0) (0, 0)]] [('hidden', '|i1'), ('communicated', '|i1')]"
       ]
      }
     ], 
     "prompt_number": 2
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "print \"a simplified JVN cell might look something like this:\"", 
      "b = np.zeros((4, 3), dtype=[(\"exc\", \"bool\"), (\"dir\", \"int8\"), (\"special\", \"bool\")])", 
      "b[0][1][\"exc\"] = True", 
      "b[0][1][\"dir\"] = 1  # maybe 1 stands for \"east\" or something", 
      "b[0][1][\"special\"] = True # special transmission", 
      "print b, b.dtype"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "a simplified JVN cell might look something like this:", 
        "[[(False, 0, False) (True, 1, True) (False, 0, False)]", 
        " [(False, 0, False) (False, 0, False) (False, 0, False)]", 
        " [(False, 0, False) (False, 0, False) (False, 0, False)]", 
        " [(False, 0, False) (False, 0, False) (False, 0, False)]] [('exc', '|b1'), ('dir', '|i1'), ('special', '|b1')]"
       ]
      }
     ], 
     "prompt_number": 3
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "print \"access via numbers is also supported:\"", 
      "print b[0][1][0], b[0][1][1], b[0][1][2]", 
      "print b[0,1][0],  b[0,1][1],  b[0,1][2]"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "access via numbers is also supported:", 
        "True 1 True", 
        "True 1 True"
       ]
      }
     ], 
     "prompt_number": 4
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "print \"but slicing along the record axis is not possible:\"", 
      "try:", 
      "    print b[0,1,0]", 
      "    raise Exception(\"unexpected success\")", 
      "except IndexError:", 
      "    print \"didn't work.\"", 
      "", 
      "print \"this doesn't work, either\"", 
      "try:", 
      "    print b[0,1,\"exc\"]", 
      "    raise Exception(\"unexpected success\")", 
      "except ValueError:", 
      "    print \"didn't work.\""
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "but slicing along the record axis is not possible:", 
        "didn't work.", 
        "this doesn't work, either", 
        "didn't work."
       ]
      }
     ], 
     "prompt_number": 5
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "print \"leaving out the last index gives us tuples\"", 
      "print b[0,0]", 
      "print b[0][0]", 
      "print", 
      "", 
      "print \"these tuples are not regular tuples, because they can be accessed with strings.\"", 
      "print type(b[0,0])", 
      "print b[0,0][\"exc\"]"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "leaving out the last index gives us tuples", 
        "(False, 0, False)", 
        "(False, 0, False)", 
        "", 
        "these tuples are not regular tuples, because they can be accessed with strings.", 
        "<type 'numpy.void'>", 
        "False"
       ]
      }
     ], 
     "prompt_number": 6
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## weave.inline considerations", 
      "", 
      "it appears, that inline cannot convert arrays with complex dtypes that have names or different types:"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "with_names = np.zeros((3, 2), dtype=[(\"foo\", \"int8\"), (\"bar\", \"int8\")])", 
      "try:", 
      "    weave.inline(\"\"\"with_names(0,1,0) = 99;\"\"\", arg_names=[\"with_names\"], type_converters=weave.converters.blitz)", 
      "except KeyError:", 
      "    print \"didn't work\""
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "didn't work"
       ]
      }
     ], 
     "prompt_number": 7
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "however, if the dtype is just \"multiples of one type\", like two 8-bit integers per field, it works"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "without_names = np.zeros((3, 2), dtype=\"2i8\")", 
      "weave.inline(\"\"\"without_names(0,1,0) = 99;\"\"\", arg_names=[\"without_names\"], type_converters=weave.converters.blitz)", 
      "print without_names"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[[ 0  0]", 
        "  [99  0]]", 
        "", 
        " [[ 0  0]", 
        "  [ 0  0]]", 
        "", 
        " [[ 0  0]", 
        "  [ 0  0]]]"
       ]
      }
     ], 
     "prompt_number": 8
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "record datatypes that are essentially the same as such a \"multiples of one type\" type can just be reshaped:"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "original = np.zeros((3, 2), dtype=[(\"foo\", \"int8\"), (\"bar\", \"int8\")])", 
      "reshaped = original.view()", 
      "reshaped.dtype = \"int8\"", 
      "reshaped.shape = (3, 2, 2)", 
      "", 
      "# reshaped being a view of original means we can change either and changes show up in both", 
      "original[0,1][\"foo\"] = 100", 
      "reshaped[2,1,1] = 99", 
      "", 
      "print original, original.dtype", 
      "print reshaped, reshaped.dtype"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[(0, 0) (100, 0)]", 
        " [(0, 0) (0, 0)]", 
        " [(0, 0) (0, 99)]] [('foo', '|i1'), ('bar', '|i1')]", 
        "[[[  0   0]", 
        "  [100   0]]", 
        "", 
        " [[  0   0]", 
        "  [  0   0]]", 
        "", 
        " [[  0   0]", 
        "  [  0  99]]] int8"
       ]
      }
     ], 
     "prompt_number": 9
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "such views can now be used in weave.inline without trouble"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "weave.inline(\"\"\"reshaped(1,1,0) = 23;\"\"\", arg_names=[\"reshaped\"], type_converters=weave.converters.blitz)", 
      "print original, original.dtype", 
      "assert original[1,1][\"foo\"] == 23"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[(0, 0) (100, 0)]", 
        " [(0, 0) (23, 0)]", 
        " [(0, 0) (0, 99)]] [('foo', '|i1'), ('bar', '|i1')]"
       ]
      }
     ], 
     "prompt_number": 10
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "accessing the array with the name of the field first we get a view", 
      "into the array that we can pass on to weave.inline"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "foo_conf = original[\"foo\"]", 
      "bar_conf = original[\"bar\"]", 
      "weave.inline(\"\"\"", 
      "    for(int i = 0; i < 3; i++) {", 
      "        foo_conf(i, 0) += 10;", 
      "    }", 
      "    for(int i = 0; i < 2; i++) {", 
      "        bar_conf(0, i) -= 23;", 
      "    }\"\"\",", 
      "    arg_names=[\"foo_conf\", \"bar_conf\"],", 
      "    type_converters=weave.converters.blitz)", 
      "print original"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[(10, -23) (100, -23)]", 
        " [(10, 0) (23, 0)]", 
        " [(10, 0) (0, 99)]]"
       ]
      }
     ], 
     "prompt_number": 11
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "this gives us the possibility to expose each entry to the array as a separate array,", 
      "which is still read-write usable.", 
      "", 
      "what is apparently not possible is giving the user access to tuples as some blitz", 
      "tiny vector or something, but that's possibly because those tiny vectors need to have the same type for each field."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "with_subshapes = np.zeros((2, 2), dtype=\"2int8\")", 
      "weave.inline(\"\"\"", 
      "    with_subshapes(1,0,0) = 99;", 
      "    \"\"\", arg_names=[\"with_subshapes\"],", 
      "    type_converters=weave.converters.blitz)", 
      "print with_subshapes"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[[ 0  0]", 
        "  [ 0  0]]", 
        "", 
        " [[99  0]", 
        "  [ 0  0]]]"
       ]
      }
     ], 
     "prompt_number": 12
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## weave conclusions", 
      "", 
      "in conclusion, we have two basic cases:", 
      "", 
      "1. all record pieces are the same type (or the type is something like \"3int8\")", 
      "2. the record pieces are different types", 
      "", 
      "in case 1, we can just turn the config into one with one more dimension,", 
      "or it already behaves like that (in the case of \"3int8\")", 
      "", 
      "in this case we can also use slicing to get the \"tuples\" at each cell.", 
      "", 
      "in case 2, however, we don't have that luxury. we have to create one", 
      "view per field and we can just expose that to the C++ code."
     ]
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## some weave toying around"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "test_types = [", 
      "        \"2int8\",", 
      "        \"(2,3)int8\",", 
      "        \"int8, int8\",", 
      "        \"int8, int16\",", 
      "        [('foo', 'int8'), ('bar', 'int8')],", 
      "        [('foo', 'int8'), ('bar', 'int16')],", 
      "    ]", 
      "", 
      "for typ in test_types:", 
      "    a = np.zeros((2, 2), dtype=typ)", 
      "    print typ, a.dtype", 
      "    print \"kind:\", a.dtype.kind", 
      "    print \"shape:\", a.dtype.shape", 
      "    print \"fields:\", a.dtype.fields", 
      "    print \"descr:\", a.dtype.descr", 
      "    print"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "2int8 int8", 
        "kind: i", 
        "shape: ()", 
        "fields: None", 
        "descr: [('', '|i1')]", 
        "", 
        "(2,3)int8 int8", 
        "kind: i", 
        "shape: ()", 
        "fields: None", 
        "descr: [('', '|i1')]", 
        "", 
        "int8, int8 [('f0', '|i1'), ('f1', '|i1')]", 
        "kind: V", 
        "shape: ()", 
        "fields: {'f0': (dtype('int8'), 0), 'f1': (dtype('int8'), 1)}", 
        "descr: [('f0', '|i1'), ('f1', '|i1')]", 
        "", 
        "int8, int16 [('f0', '|i1'), ('f1', '<i2')]", 
        "kind: V", 
        "shape: ()", 
        "fields: {'f0': (dtype('int8'), 0), 'f1': (dtype('int16'), 1)}", 
        "descr: [('f0', '|i1'), ('f1', '<i2')]", 
        "", 
        "[('foo', 'int8'), ('bar', 'int8')] [('foo', '|i1'), ('bar', '|i1')]", 
        "kind: V", 
        "shape: ()", 
        "fields: {'foo': (dtype('int8'), 0), 'bar': (dtype('int8'), 1)}", 
        "descr: [('foo', '|i1'), ('bar', '|i1')]", 
        "", 
        "[('foo', 'int8'), ('bar', 'int16')] [('foo', '|i1'), ('bar', '<i2')]", 
        "kind: V", 
        "shape: ()", 
        "fields: {'foo': (dtype('int8'), 0), 'bar': (dtype('int16'), 1)}", 
        "descr: [('foo', '|i1'), ('bar', '<i2')]", 
        ""
       ]
      }
     ], 
     "prompt_number": 13
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "", 
      "", 
      "class RecordAccessor(object):", 
      "    def __init__(self, dtype):", 
      "        self.descr = list(dtype.descr)", 
      "        # so what goes here ...?"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 14
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## display considerations", 
      "", 
      "reshaping and views give us a nice way to palettize such multi-part arrays. like this:"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "multipart = np.zeros((3, 3), dtype=[(\"first_bit\", \"bool\"), (\"second_bit\", \"bool\")])", 
      "multipart[\"first_bit\"].view().reshape(9)[:]  = np.array([True,  False, True, False, True, True,  False, True, True])", 
      "multipart[\"second_bit\"].view().reshape(9)[:] = np.array([False, False, True, True,  True, False, False, False, True])", 
      "print multipart", 
      "", 
      "display = multipart.view()", 
      "display.dtype=\"int16\"", 
      "display.shape=(3,3)", 
      "print display"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[(True, False) (False, False) (True, True)]", 
        " [(False, True) (True, True) (True, False)]", 
        " [(False, False) (True, False) (True, True)]]", 
        "[[  1   0 257]", 
        " [256 257   1]", 
        " [  0   1 257]]"
       ]
      }
     ], 
     "prompt_number": 11
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "this configuration is limited to the values 0, 1, 256 and 257. It could now theoretically just be viewed with a normal zasim based config painter:"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "from zasim.display.qt import render_state_array, qimage_to_pngstr, make_gray_palette", 
      "", 
      "new_palette,_ = make_gray_palette([0, 1, 256, 257])", 
      "publish_png(qimage_to_pngstr(render_state_array(display.T, new_palette).scaled(90, 90)))"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "display_data", 
       "png": "iVBORw0KGgoAAAANSUhEUgAAAFoAAABaCAIAAAC3ytZVAAAAA3NCSVQICAjb4U/gAAAACXBIWXMA\nAA7EAAAOxAGVKw4bAAAA30lEQVR4nO3asREDIRAEQaEcnyAhyJfdAZyhqukA1pg6D9bzPJ8Z996h\n5fd9h5a/Q7t/qhwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB8qB\ncmCdc4am5/4G7L2HlrsOlAPlQDlQDpQD5UA5UA6UA+VAOVAOlAPlQDlQDpQD5UA5UA6UA+VAOVAO\nlAPlQDlQDqy56bkX/LlfB10HyoFyoBwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB8qBcqAcKAfKgXKg\nHCgHyoFyoBwoB348AQ2rWQsa2QAAAABJRU5ErkJggg==\n"
      }
     ], 
     "prompt_number": 12
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "reshaping and views gets a bit more complicated if the dtypes are not the same for each field,", 
      "but since most dtypes are just padded up to multiples of bytes or something,", 
      "we can just calculate the next bigger integer type and use that."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "example_dtypes = [\"int8, bool\", \"2int8, int16\", \"int8, int16, bool\"]", 
      "for the_string in example_dtypes:", 
      "    print \"{}: {}\".format(the_string, np.dtype(the_string).itemsize)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "int8, bool: 2", 
        "2int8, int16: 4", 
        "int8, int16, bool: 4"
       ]
      }
     ], 
     "prompt_number": 13
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## automatic reshaping and palette generation", 
      "", 
      "since the dtypes can just be figured out from the first value we get,", 
      "all we really need to create a palette is a range of allowed values", 
      "for each of the fields and then we can generate a palette for that, much like this:"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "example_dtypes_2 = example_dtypes + \\", 
      "    [[(\"up\", \"bool\"), (\"left\", \"bool\")],", 
      "     [(\"one\", \"int8\"), (\"two\", \"bool\")],", 
      "     ]", 
      "for the_string in example_dtypes_2:", 
      "    print the_string", 
      "    dt = np.dtype(the_string)", 
      "    print dt.fields", 
      "    print \"sizes:\", {fn: (x.name, x.itemsize) for fn, (x, offset) in dt.fields.iteritems()}", 
      "    print \"sum of sizes:\", sum(x.itemsize for x, offset in dt.fields.values())", 
      "    print"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "int8, bool", 
        "{'f0': (dtype('int8'), 0), 'f1': (dtype('bool'), 1)}", 
        "sizes: {'f0': ('int8', 1), 'f1': ('bool', 1)}", 
        "sum of sizes: 2", 
        "", 
        "2int8, int16", 
        "{'f0': (dtype(('int8',(2,))), 0), 'f1': (dtype('int16'), 2)}", 
        "sizes: {'f0': ('void16', 2), 'f1': ('int16', 2)}", 
        "sum of sizes: 4", 
        "", 
        "int8, int16, bool", 
        "{'f0': (dtype('int8'), 0), 'f1': (dtype('int16'), 1), 'f2': (dtype('bool'), 3)}", 
        "sizes: {'f0': ('int8', 1), 'f1': ('int16', 2), 'f2': ('bool', 1)}", 
        "sum of sizes: 4", 
        "", 
        "[('up', 'bool'), ('left', 'bool')]", 
        "{'up': (dtype('bool'), 0), 'left': (dtype('bool'), 1)}", 
        "sizes: {'up': ('bool', 1), 'left': ('bool', 1)}", 
        "sum of sizes: 2", 
        "", 
        "[('one', 'int8'), ('two', 'bool')]", 
        "{'two': (dtype('bool'), 1), 'one': (dtype('int8'), 0)}", 
        "sizes: {'two': ('bool', 1), 'one': ('int8', 1)}", 
        "sum of sizes: 2", 
        ""
       ]
      }
     ], 
     "prompt_number": 14
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "from itertools import product", 
      "from binascii import hexlify", 
      "", 
      "def pad_dtype(arrdtype):", 
      "    \"\"\"calculates a new dtype that is padded to fit each of the dtypes cells into one integer.", 
      "", 
      "    :returns: the new dtype and value ranges for the padding fields.\"\"\"", 
      "", 
      "    bytesize = arrdtype.itemsize", 
      "", 
      "    available_bytesizes = [1, 2, 4, 8, 16, 32, 64]", 
      "    # the first higher one we can take, we do take.", 
      "    try:", 
      "        new_bytesize = [bs for bs in available_bytesizes if bs >= bytesize][0]", 
      "    except IndexError:", 
      "        raise ValueError(\"There is no dtype large enough to encompass one field :(\")", 
      "", 
      "    if new_bytesize != bytesize:", 
      "        print \"bytesizes differ: {} -> {}\".format(bytesize, new_bytesize)", 
      "        # if the bytesizes differ, we have to pad the data up!", 
      "        pad_size = new_bytesize - bytesize", 
      "", 
      "        # turn pad_size into a binary like 1101. ones tell us which sizes we need", 
      "        # for padding", 
      "        pad_binary = bin(pad_size)[2:]", 
      "        take_padders = [size + 1 for size, take in enumerate(pad_binary) if take == \"1\"]", 
      "", 
      "        possible_value_dict = {}", 
      "        new_dtype_descrs = []", 
      "        for padval in take_padders:", 
      "            fieldname = \"padding_\" + str(padval)", 
      "            typename = \"i\" + str(padval)", 
      "            shape = ()", 
      "            new_dtype_descrs.append((fieldname, typename, shape))", 
      "            possible_value_dict[fieldname] = [0]", 
      "", 
      "        new_dtype_descrs.extend(arrdtype.descr)", 
      "", 
      "        print \"the result of the padding operation is:\"", 
      "        print new_dtype_descrs", 
      "        new_dtype = np.dtype(new_dtype_descrs)", 
      "", 
      "        return new_dtype, possible_value_dict", 
      "", 
      "    return arrdtype, {}", 
      "", 
      "def analyze_dtype(arrdtype, possible_value_dict):", 
      "    \"\"\"generates keys for a palette from thegiven array.\"\"\"", 
      "", 
      "    # generate an array of all possibilities", 
      "    sizes = [len(possible_value_dict[descr[0]]) for descr in arrdtype.descr]", 
      "", 
      "    all_values = np.zeros(shape=sizes, dtype=arrdtype)", 
      "", 
      "    value_ranges_per_subfield = [possible_value_dict[descr[0]] for descr in arrdtype.descr]", 
      "", 
      "    positions = product(*map(range, sizes))", 
      "    for position in positions:", 
      "        values = [value_ranges_per_subfield[idx][position[idx]] for idx in range(len(sizes))]", 
      "        all_values[tuple(position)] = tuple(values)", 
      "", 
      "    print \"these are all possible values:\"", 
      "    print all_values", 
      "", 
      "    # now change the dtype", 
      "", 
      "    new_array = all_values.view()", 
      "    new_array.dtype = \"i\" + str(arrdtype.itemsize)", 
      "", 
      "    return new_array.reshape(reduce(lambda a, b: a * b, sizes))", 
      "", 
      "test_dtype = np.dtype(\"int8, int16\")", 
      "print \"we're going to analyse this dtype:\"", 
      "print test_dtype", 
      "print", 
      "", 
      "possible_values = {\"f0\": [0, 1, 2, 3],", 
      "                   \"f1\": [1, 2, 4, 8, 16]}", 
      "", 
      "test_dtype, possible_values_addendum = pad_dtype(test_dtype)", 
      "possible_values.update(possible_values_addendum)", 
      "", 
      "print \"we've decided, that the following values are possible for the fields:\"", 
      "print possible_values", 
      "print", 
      "", 
      "all_values = analyze_dtype(test_dtype, possible_values)", 
      "print \"these are all values possible, as integers\"", 
      "print all_values", 
      "print", 
      "", 
      "print \"the bytes are packed like this:\"", 
      "chunksize = len(all_values.data) / len(all_values)", 
      "for chunk in range(len(all_values)):", 
      "    print hexlify(str(all_values.data)[chunk * chunksize:(chunk + 1) * chunksize])"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "we're going to analyse this dtype:", 
        "[('f0', '|i1'), ('f1', '<i2')]", 
        "", 
        "bytesizes differ: 3 -> 4", 
        "the result of the padding operation is:", 
        "[('padding_1', 'i1', ()), ('f0', '|i1'), ('f1', '<i2')]", 
        "we've decided, that the following values are possible for the fields:", 
        "{'f0': [0, 1, 2, 3], 'f1': [1, 2, 4, 8, 16], 'padding_1': [0]}", 
        "", 
        "these are all possible values:", 
        "[[[(0, 0, 1) (0, 0, 2) (0, 0, 4) (0, 0, 8) (0, 0, 16)]", 
        "  [(0, 1, 1) (0, 1, 2) (0, 1, 4) (0, 1, 8) (0, 1, 16)]", 
        "  [(0, 2, 1) (0, 2, 2) (0, 2, 4) (0, 2, 8) (0, 2, 16)]", 
        "  [(0, 3, 1) (0, 3, 2) (0, 3, 4) (0, 3, 8) (0, 3, 16)]]]", 
        "these are all values possible, as integers", 
        "[  65536  131072  262144  524288 1048576   65792  131328  262400  524544", 
        " 1048832   66048  131584  262656  524800 1049088   66304  131840  262912", 
        "  525056 1049344]", 
        "", 
        "the bytes are packed like this:", 
        "00000100", 
        "00000200", 
        "00000400", 
        "00000800", 
        "00001000", 
        "00010100", 
        "00010200", 
        "00010400", 
        "00010800", 
        "00011000", 
        "00020100", 
        "00020200", 
        "00020400", 
        "00020800", 
        "00021000", 
        "00030100", 
        "00030200", 
        "00030400", 
        "00030800", 
        "00031000"
       ]
      }
     ], 
     "prompt_number": 15
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "we can now turn any old array into a padded one that we can interpret as integers"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "def pad_array_and_copy(arr):", 
      "    new_dtype, _ = pad_dtype(arr.dtype)", 
      "    print new_dtype", 
      "    print arr.dtype", 
      "    if new_dtype != arr.dtype:", 
      "        # we have to copy data over because of padding", 
      "        new_arr = np.empty(arr.shape, dtype=new_dtype)", 
      "        for descr in arr.dtype.descr:", 
      "            fn = descr[0]", 
      "            # just copy all data from the fields over", 
      "            new_arr[fn] = arr[fn]", 
      "    else:", 
      "        new_arr = arr.copy()", 
      "", 
      "    return new_arr", 
      "", 
      "def reinterpret_as_integers(arr):", 
      "    return arr.view(\"i{}\".format(arr.dtype.itemsize))", 
      "", 
      "def uninterpret(arr, orig_dtype):", 
      "    view = arr.view()", 
      "    view.dtype = orig_dtype", 
      "    return view"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 16
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "test = np.zeros((2, 2), dtype=(\"int8, int8\"))", 
      "test[\"f0\"] = [[0, 1], [2, 3]]", 
      "test[\"f1\"] = [[3, 1], [0, 2]]", 
      "other = pad_array_and_copy(test)", 
      "print reinterpret_as_integers(other)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[('f0', '|i1'), ('f1', '|i1')]", 
        "[('f0', '|i1'), ('f1', '|i1')]", 
        "[[768 257]", 
        " [  2 515]]"
       ]
      }
     ], 
     "prompt_number": 17
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "now we can turn that data into a palette and draw a bit of stuff with it.", 
      "", 
      "the palette version of the random config generator should accept our \"all_values\" list."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "new_palette,_ = make_gray_palette(list(all_values))", 
      "", 
      "from zasim.config import RandomConfigurationFromPalette", 
      "", 
      "display = RandomConfigurationFromPalette(list(all_values)).generate((5, 5))", 
      "", 
      "publish_png(qimage_to_pngstr(render_state_array(display.T, new_palette).scaled(90, 90)))", 
      "", 
      "print uninterpret(display, test_dtype)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "display_data", 
       "png": "iVBORw0KGgoAAAANSUhEUgAAAFoAAABaCAIAAAC3ytZVAAAAA3NCSVQICAjb4U/gAAAACXBIWXMA\nAA7EAAAOxAGVKw4bAAABDUlEQVR4nO3bsY2DQBRF0fXKndGA+0BugAKgKVwAXW18Qv+AYHVP/jTS\n1UQjeFzX9fO9z+dz22rf99vO+h1s/rFyoBwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB8qBx7Isg9md\nL1Qzr9drsOp2oBwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB8qBcuA5e9e60+y97v1+D1bdDpQD5UA5\nUA6UA+VAOVAOlAPlQDlQDpQD5UA5UA4Mvw27c7Vt22B1nudg1e1AOVAOlAPlQDlQDpQD5UA5UA6U\nA+VAOVAOlAPlwHM2m/3zuK7r7LiB4zgGq24HyoFyoBwoB8qBcqAcKAfKgXKgHCgHyoFyoBwoB/4A\nuiUfdpg0/9kAAAAASUVORK5CYII=\n"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "[[(0, 3, 1) (0, 2, 1) (0, 1, 1) (0, 3, 1) (0, 2, 1)]", 
        " [(0, 1, 1) (0, 3, 1) (0, 2, 1) (0, 2, 1) (0, 3, 2)]", 
        " [(0, 3, 1) (0, 3, 1) (0, 2, 1) (0, 3, 1) (0, 1, 1)]", 
        " [(0, 1, 1) (0, 2, 1) (0, 3, 1) (0, 1, 1) (0, 3, 16)]", 
        " [(0, 1, 1) (0, 1, 1) (0, 3, 8) (0, 1, 1) (0, 2, 1)]]"
       ]
      }
     ], 
     "prompt_number": 18
    }
   ]
  }
 ]
}