Permalink
Browse files

Added `query%` macro to LINQ to SQL

  • Loading branch information...
1 parent 100ac49 commit 7f3cc42bf40c01bab5db31a3ecb5755233a8da7e @lihaoyi committed May 8, 2013
Showing with 115 additions and 95 deletions.
  1. +4 −24 macropy/demo.py
  2. +54 −52 macropy/macros2/linq.py
  3. +18 −0 macropy/macros2/linq_test.py
  4. +39 −19 readme.md
View
@@ -1,36 +1,16 @@
from sqlalchemy import *
-from macropy.macros2.linq import macros, sql, generate_schema
+from macropy.macros2.linq import macros, sql, query, generate_schema
engine = create_engine("sqlite://")
for line in open("macros2/linq_test_dataset.sql").read().split(";"):
engine.execute(line.strip())
db = generate_schema(engine)
-query = sql%(
- x.name for x in db.bbc
- if x.gdp / x.population > (
- y.gdp / y.population for y in db.bbc
- if y.name == 'United Kingdom'
- )
- if (x.region == 'Europe')
-)
-
-query = select([db.bbc.c.name]).where(
- db.bbc.c.gdp / db.bbc.c.population > select(
- [(db.bbc.c.gdp / db.bbc.c.population)]
- ).where(
- db.bbc.c.name == 'United Kingdom'
- )
-).where(
- db.bbc.c.region == 'Europe'
-)
-results = engine.execute(query).fetchall()
-
-print query
-
-for line in results: print line
+query_string = sql%((x.name, x.area) for x in db.bbc if x.area > 10000000)
+print type(query_string)
+print query_string
"""
Demos
@@ -66,81 +66,83 @@
macros = Macros()
@macros.expr
def sql(tree):
- def recurse(tree, scope):
- if type(tree) is Compare and type(tree.ops[0]) is In:
- return q%(ast%recurse(tree.left, scope)).in_(ast%recurse(tree.comparators[0], scope))
-
- if type(tree) is Compare:
- tree.left = recurse(tree.left, scope)
- tree.comparators = map(f%recurse(_, scope), tree.comparators)
- return tree
+ x = recurse(tree, [])
+ return x
- if type(tree) is Call:
- tree.func = recurse(tree.func, scope)
- tree.args = map(f%recurse(_, scope), tree.args)
- return tree
+@macros.expr
+def query(tree):
+ x = recurse(tree, [])
+ return q%(lambda query: query.bind.execute(query).fetchall())(ast%x)
- if type(tree) is BinOp:
- tree.left = recurse(tree.left, scope)
- tree.right = recurse(tree.right, scope)
- return tree
+def recurse(tree, scope):
+ if type(tree) is Compare and type(tree.ops[0]) is In:
+ return q%(ast%recurse(tree.left, scope)).in_(ast%recurse(tree.comparators[0], scope))
- if type(tree) is BoolOp:
- tree.values = map(f%recurse(_, scope), tree.values)
- return tree
+ if type(tree) is Compare:
+ tree.left = recurse(tree.left, scope)
+ tree.comparators = map(f%recurse(_, scope), tree.comparators)
+ return tree
- if type(tree) is Tuple:
- tree.elts = map(f%recurse(_, scope), tree.elts)
+ if type(tree) is Call:
+ tree.func = recurse(tree.func, scope)
+ tree.args = map(f%recurse(_, scope), tree.args)
+ return tree
- if type(tree) is Attribute:
+ if type(tree) is BinOp:
+ tree.left = recurse(tree.left, scope)
+ tree.right = recurse(tree.right, scope)
+ return tree
- tree.value = recurse(tree.value, scope)
- return tree
+ if type(tree) is BoolOp:
+ tree.values = map(f%recurse(_, scope), tree.values)
+ return tree
- if type(tree) is GeneratorExp:
+ if type(tree) is Tuple:
+ tree.elts = map(f%recurse(_, scope), tree.elts)
- aliases = map(f%_.target, tree.generators)
- tables = map(f%_.iter, tree.generators)
- import random
+ if type(tree) is Attribute:
- aliased_tables = map(lambda x: q%((ast%x).alias().c), tables)
+ tree.value = recurse(tree.value, scope)
+ return tree
- ifs = [
- recurse(ifcond, None)
- for gen in tree.generators
- for ifcond in gen.ifs
- ]
+ if type(tree) is GeneratorExp:
- elt = tree.elt
- if type(elt) is Tuple:
+ aliases = map(f%_.target, tree.generators)
+ tables = map(f%_.iter, tree.generators)
+ import random
- sel = q%(ast_list%recurse(elt, None).elts)
- else:
- sel = q%[ast%recurse(elt, None)]
+ aliased_tables = map(lambda x: q%((ast%x).alias().c), tables)
+ ifs = [
+ recurse(ifcond, None)
+ for gen in tree.generators
+ for ifcond in gen.ifs
+ ]
+ elt = tree.elt
+ if type(elt) is Tuple:
- out = q%select(ast%sel)
+ sel = q%(ast_list%recurse(elt, None).elts)
+ else:
+ sel = q%[ast%recurse(elt, None)]
- for cond in ifs:
- out = q%(ast%out).where(ast%cond)
- if scope != []:
- out = q%(ast%out).as_scalar()
+ out = q%select(ast%sel)
- out = q%(lambda x: ast%out)()
- out.func.args.args = aliases
- out.args = aliased_tables
- return out
- return tree
+ for cond in ifs:
+ out = q%(ast%out).where(ast%cond)
- x = recurse(tree, [])
- print unparse_ast(x)
- return x
+ if scope != []:
+ out = q%(ast%out).as_scalar()
+ out = q%(lambda x: ast%out)()
+ out.func.args.args = aliases
+ out.args = aliased_tables
+ return out
+ return tree
def generate_schema(engine):
metadata = sqlalchemy.MetaData(engine)
metadata.reflect()
@@ -144,6 +144,24 @@ def test_aliased(self):
)
)
+ def test_query_macro(self):
+ query = sql%(
+ func.distinct(x.region) for x in db.bbc
+ if (
+ func.sum(w.population) for w in db.bbc
+ if w.region == x.region
+ ) > 100000000
+ )
+ sql_results = engine.execute(query).fetchall()
+ query_macro_results = query%(
+ func.distinct(x.region) for x in db.bbc
+ if (
+ func.sum(w.population) for w in db.bbc
+ if w.region == x.region
+ ) > 100000000
+ )
+ assert sql_results == query_macro_results
+
def test_join(self):
compare_queries(
"""
View
@@ -535,34 +535,53 @@ LINQ to SQL
```python
db = generate_schema(engine)
-query = sql%((x.name, x.area) for x in db.bbc if x.area > 10000000)
-results = engine.execute(query).fetchall()
-
-print query
-# SELECT bbc_1.name, bbc_1.area
-# FROM bbc AS bbc_1
-# WHERE bbc_1.area > ?
-
+results = query%(
+ x.name for x in db.bbc
+ if x.gdp / x.population > (
+ y.gdp / y.population for y in db.bbc
+ if y.name == 'United Kingdom'
+ )
+ if (x.region == 'Europe')
+)
for line in results: print line
-# (u'Russia', 17000000)
+# (u'Denmark',)
+# (u'Iceland',)
+# (u'Ireland',)
+# (u'Luxembourg',)
+# (u'Norway',)
+# (u'Sweden',)
+# (u'Switzerland',)
```
-This feature is inspired by [C#'s LINQ to SQL](http://msdn.microsoft.com/en-us/library/bb386976.aspx). In short, code used to manipulate lists is lifted into an AST which is then cross-compiled into a snippet of [SQL](http://en.wikipedia.org/wiki/SQL). This preserves the manipulation, but instead of performing it locally on some data structure, sends the query to a remote database to be performed there.
+This feature is inspired by [C#'s LINQ to SQL](http://msdn.microsoft.com/en-us/library/bb386976.aspx). In short, code used to manipulate lists is lifted into an AST which is then cross-compiled into a snippet of [SQL](http://en.wikipedia.org/wiki/SQL). In this case, it is the `query%` macro which does this lifting and cross-compilation. Instead of performing the manipulation locally on some data structure, the compiled query is sent to a remote database to be performed there.
+
+This allows you to write queries to a database in the same way you would write queries on in-memory lists, which is really very nice. The translation is a relatively thin layer of over the [SQLAlchemy](http://www.sqlalchemy.org/) Query Language, which does the heavy lifting of converting the query into a raw SQL string:. If we start with a simple query
-This allows you to write queries to a database in the same way you would write queries on in-memory lists, which is really very nice. Although this implementation is just a thin facade over [SQLAlchemy](http://www.sqlalchemy.org/), compare the query above:
+```python
+print query%((x.name, x.area) for x in db.bbc if x.area > 10000000)
+# [(u'Russia', 17000000)]
+```
+
+This is to the equivalent SQLAlchemy query:
```python
-sql%((x.name, x.area) for x in db.bbc if x.area > 10000000)
+print engine.execute(select([bbc.c.name, bbc.c.area]).where(bbc.c.area > 10000000)).fetchall()
```
-to the equivalent SQLAlchemy query:
+To verify that LINQ to SQL is actually cross-compiling the python to SQL, and not simply requesting everything and performing the manipulation locally, we can use the `sql%` macro to perform the lifting of the query without executing it:
```python
-select([bbc.c.name, bbc.c.area]).where(bbc.c.area > 10000000)
+query_string = sql%((x.name, x.area) for x in db.bbc if x.area > 10000000)
+print type(query_string)
+# <class 'sqlalchemy.sql.expression.Select'>
+print query_string
+# SELECT bbc_1.name, bbc_1.area
+# FROM bbc AS bbc_1
+# WHERE bbc_1.area > ?
```
-The SQLAlchemy query looks pretty odd, for somebody who knows python but isn't familiar with the library. This is because SQLAlchemy cannot "lift" Python code into an AST to manipulate, and instead have to construct the AST manually using python objects. Although it works pretty well, the syntax and semantics of the queries is completely different from python. In more complex examples, the way the semantics of an SQLAlchemy query diverge from normal python becomes much more pronounced.
+As we can see, LINQ to SQL converts the python list-comprehension into a SQLAlchemy `Select`, which when stringified becomes a valid SQL string.
Consider a less trivial example: we want to find all countries in europe who have a GDP per Capita greater than the United Kingdom. This is the SQLAlchemy code to do so:
@@ -578,11 +597,11 @@ query = select([db.bbc.c.name]).where(
)
```
-Already we are bumping into edge cases: the `db.bbc` in the nested query is referred to the same way as the `db.bbc` in the outer query, although they are clearly different! One may wonder, what if, in the inner query, we wish to refer to the outer query's values?
+The SQLAlchemy query looks pretty odd, for somebody who knows python but isn't familiar with the library. This is because SQLAlchemy cannot "lift" Python code into an AST to manipulate, and instead have to construct the AST manually using python objects. Although it works pretty well, the syntax and semantics of the queries is completely different from python.
-Naturally, there will be solutions to all of these requirements. In the end, SQLAlchemy ends up effectively creating its own programming language, with its own scoping, its own name binding, etc..
+Already we are bumping into edge cases: the `db.bbc` in the nested query is referred to the same way as the `db.bbc` in the outer query, although they are clearly different! One may wonder, what if, in the inner query, we wish to refer to the outer query's values? Naturally, there will be solutions to all of these requirements. In the end, SQLAlchemy ends up effectively creating its own mini programming language, with its own concept of scoping, name binding, etc., basically duplicating what Python already has but with messier syntax.
-In the equivalent LINQ code, the scoping of which `db.bbc` you are referring to is much more explicit, and closely follows Python's scoping rules:
+In the equivalent LINQ code, the scoping of which `db.bbc` you are referring to is much more explicit, and in general the semantics are identical to a typical python comprehension:
```python
query = sql%(
@@ -618,7 +637,8 @@ for line in results: print line
# (u'Sweden',)
# (u'Switzerland',)
```
-This clone of LINQ to SQL still does not support the vast capabilities of the SQL language. Nevertheless, it demonstrates how easy it is to use macros to lift python snippets into an AST and cross-compile it into another language.
+
+This clone of LINQ to SQL still does not support the vast capabilities of the SQL language. Nevertheless, it demonstrates how easy it is to use macros to lift python snippets into an AST and cross-compile it into another language, and how nice the syntax and semantics can be for these embedded DSLs.
Quick Lambdas
-------------

0 comments on commit 7f3cc42

Please sign in to comment.