Merge pull request #62 from Quansight-Labs/expand-expressions

Add support for kwargs, variable args, and conversion

.coveragerc

@@ -2,6 +2,7 @@
 branch = True
 source = 
     metadsl
+    metadsl_core
 [report]
 omit =
     *_test.py

.gitignore

@@ -19,4 +19,5 @@ htmlcov
 .ipython
 .jupyter
 .cache
-.local
+.local
+.testmondata

Demo.ipynb

@@ -0,0 +1,261 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from __future__ import annotations\n",
+    "\n",
+    "from metadsl import *\n",
+    "from metadsl_core import *\n",
+    "from metadsl_visualize import *\n",
+    "import metadsl_core.vec\n",
+    "set_rule(core_rules)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# `metadsl` Demo\n",
+    "\n",
+    "In this notebook we show a few examples of using `metadsl`. The outputs are interactive widgets that show the progress of replacing the expressions. You can drag the slider to move from the original expression to the final replaced one"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Indexing a vector"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can create a vector type and then index it, to see the conversion progress:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "36999a21c45047afad94237d3d4e011e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(IntSlider(value=0, max=1), Output()))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "Vec.create(Integer.from_int(1), Integer.from_int(2))[Integer.from_int(0)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If we look at how this is implemented, we see how we define a rule to replace indexing a vector:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[0;31mSignature:\u001b[0m       \u001b[0mmetadsl_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'int'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'typing.Sequence[T]'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;34m'R[T]'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+       "\u001b[0;31mCall signature:\u001b[0m  \u001b[0mmetadsl_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mIterable\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmetadsl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mReplacement\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+       "\u001b[0;31mType:\u001b[0m            MatchRule\n",
+       "\u001b[0;31mString form:\u001b[0m     metadsl_core.vec.getitem\n",
+       "\u001b[0;31mFile:\u001b[0m            ~/p/metadsl/metadsl_core/vec.py\n",
+       "\u001b[0;31mSource:\u001b[0m         \n",
+       "\u001b[0;34m@\u001b[0m\u001b[0mregister\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\n",
+       "\u001b[0;34m\u001b[0m\u001b[0;34m@\u001b[0m\u001b[0mrule\u001b[0m\u001b[0;34m\u001b[0m\n",
+       "\u001b[0;34m\u001b[0m\u001b[0;32mdef\u001b[0m \u001b[0mgetitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequence\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n",
+       "\u001b[0;34m\u001b[0m    \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mVec\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mInteger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+       "\u001b[0;31mClass docstring:\u001b[0m\n",
+       "Creates a replacement rule given a function that maps from wildcard inputs\n",
+       "to two things, a template expression tree and a replacement thunk.\n",
+       "\n",
+       "If the template matches an expression, it will be replaced with the result of the thunk, replacing\n",
+       "the input args with the nodes at their locations in the template.\n",
+       "\n",
+       "You can also return None from the rule to signal that it won't match.\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "metadsl_core.vec.getitem??"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## indexing an array and conversion"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we can try creating some NumPy arrays and indexing them. We see that through the replacement system we figure out if we are indexing with a tuple or an integer. This is an easier place to compile to different backends (like LLVM) than just the raw NumPy calls:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "15702498ee3c4463a06a28cb7c694c24",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(IntSlider(value=0, max=28), Output()))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "arange(10)[5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0d7fac3bab474d52ab57a6b158829ca5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(IntSlider(value=0, max=43), Output()))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "arange(10)[(2,)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we can show one way of compiling these calls, by replacing them with the corresponding NumPy calls, to compute the results:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "762d9f4a08d445d998369c3960229479",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(IntSlider(value=0, max=36), Output()))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "unbox_ndarray_compat(arange(10)[5])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "341f15f1fc1147f6bb0ad9c2387f119b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(IntSlider(value=0, max=52), Output()))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "unbox_ndarray_compat(arange(10)[(2,)])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this demo, we show how we can break up the NumPy API into different layers, all of which are extensible:\n",
+    "\n",
+    "1. A compatibility layer that works like the existing NumPy API, except isn't limited to the Python types of the current API\n",
+    "2. A type safe version of this API. The conversion between the compatability layer and this layer is extensible, so that third party authors can add new conversion between their own Python objects and the typed representation.\n",
+    "3. (Not implemented yet) A mathematical representation of the array operations that generalizes the api to a much smaller subset of functions.\n",
+    "4. A backend layer that translates either back to Python calls or source code, or to other targets like LLVM or Tensorflow.\n",
+    "\n",
+    "The key is that all these layers are composable, so you could have different frontends for any of them or add your own. This is all done through a typed replacement system that is compatible with static analysis using MyPy."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

README.md

@@ -1,59 +1,6 @@
 # `metadsl`
 
-[![Documentation Status](https://readthedocs.org/projects/metadsl/badge/?version=latest)](https://metadsl.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/metadsl/badge/?version=latest)](https://metadsl.readthedocs.io/en/latest/?badge=latest) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Quansight-Labs/metadsl/master?urlpath=lab/tree/Demo.ipynb)
 
 A framework for creating domain specific language libraries in Python.
 
-
-## Guiding Principles
-
-The goal here is to share as much optimization and representation logic as possible, so that the world users
-exist in can be extended more easily without having to cause them to change their code.
-
-For example, if someone comes up with a new way of executing linear algebra or wants to try out a new way of optimizing
-expressions, they should be able to write those things in a pluggable manner, so that users can try them out with minimal
-effort. 
-
-This means we have to explicitly expose the protocols of the different levels to foster distributed collaboration and reuse. 
-
-## Development
-
-Either use repo2docker:
-
-```bash
-repo2docker -E .
-```
-
-
-Or get started with Conda/flit:
-
-```bash
-conda create -n metadsl jupyterlab
-conda activate metadsl
-pip install flit
-flit install --symlink
-```
-
-### Tests
-
-This runs mypy and tests, and reports coverage.
-
-```bash
-pytest --cov
-# open coverage file
-open htmlcov/index.html
-```
-
-You can also test that the documentation notebooks run correctly, but this
-[must be run separately from the code coverage](https://github.com/computationalmodelling/nbval/issues/116):
-
-```bash
-pytest docs/*.ipynb --nbval
-````
-
-
-### Docs
-
-```bash
-sphinx-autobuild docs docs/_build/html/
-```

binder/postBuild

@@ -1 +1,4 @@
 flit install --symlink
+flit -f core.pyproject.toml install --symlink
+flit -f visualize.pyproject.toml install --symlink
+jupyter labextension install @jupyter-widgets/jupyterlab-manager@0.38.0

core.pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["flit"]
+build-backend = "flit.buildapi"
+
+[tool.flit.metadata]
+module = "metadsl_core"
+author = "Saul Shanabrook"
+author-email = "s.shanabrook@gmail.com"
+home-page = "https://github.com/Quansight-Labs/metadsl"
+requires = [
+    "metadsl",
+]
+requires-python = ">=3.7"
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]