Merge branch 'master' into bmyerz/tpc-h-fixes

Conflicts:
	c_test_environment/grappa_ingest.py
	raco/backends/cpp/cpp.py
	raco/backends/radish/radish.py

README.md

@@ -6,6 +6,8 @@ A pure Python compiler and optimization framework for relational algebra.
 [![Build Status](https://travis-ci.org/uwescience/raco.png?branch=master)](https://travis-ci.org/uwescience/raco)
 [![Coverage Status](https://coveralls.io/repos/uwescience/raco/badge.png)](https://coveralls.io/r/uwescience/raco)
 
+Raco takes as input a number of source languages and has a growing number of output languages.
+
 Source languages include:
 
 * Datalog
@@ -14,12 +16,12 @@ Source languages include:
 
 Output languages include:
 
-* Logical relational algebra
+* Logical relational algebra (+ while loop)
 * The Myria physical algebra (in JSON)
-* A C++ algebra, C++ source code
-* A Grappa algebra, Grappa source code
-* A pseudocode algebra
-* A Python algebra
+* C++ physical algebra, C++ source code
+* [Grappa](http://grappa.io) (distributed C++) physical algebra, Grappa source code
+* Pseudocode algebra
+* Python physical algebra
 * SPARQL (partial)
 
 Users can of course author programs by directly instantiating one of the intermediate or output algebras as well as one of the source languages.
@@ -44,15 +46,27 @@ python setup.py install
 
 # Run tests
 
-To execute the tests, run `nosetests` in the root directory of the repository. See `nosetests -h` for more options or consult the [nose documentation](https://nose.readthedocs.org).
+To execute the tests, run `nosetests` in the root directory of the repository. 
+
+A few hints: to print test names use `-v`
+```
+nosetests -v
+```
+And fail on first error use `-x`
+```
+nosetests -x -v
+```
+
+See `nosetests -h` for more options or consult the [nose documentation](https://nose.readthedocs.org).
+
 
 #### Requirements for C++ backend tests
-- C++11 compiler
+- C++11 compiler, i.e. gcc-4.7 or later, clang-3.3 or later
 - sqlite3
 
 # Example
 
-We are currently using Raco mostly for Myria. To try parsing and understanding a program written in the Myria language, use the included `myrial` utility.
+Raco is the compiler for Myria. To try parsing and understanding a program written in the Myria language, use the included `myrial` utility.
 
 Note that the commands below run the `myrial` utility from the included `scripts` directory. However, the install command above will in fact install `myrial` in your `$PATH`.
 
@@ -102,25 +116,45 @@ Sequence
 ### Visualize a Myria plan as a graph
 Pass the `-d` option to `scripts/myrial`. Output omitted for brevity.
 
+### Output the Myria physical plan as json (what MyriaX understands)
+Pass the `-j` option to `scripts/myrial`. Output omitted for brevity.
+
 # C++ and Grappa output (Radish)
-There is also Grappa output for Raco.
+Raco has backend compilers that emit C++ and [Grappa](http://grappa.io) programs for query plans. To find out how it works in depth, read the [UW technical report](https://homes.cs.washington.edu/~bdmyers/papers/UW-CSE-14-10-01.pdf).
+
+### Output C++ plan and source program
+```
+# generate the query and save to join.cpp
+scripts/myrial --cpp examples/join.myl
+
+# build
+mv join.cpp c_test_environment/
+cd c_test_environment; make join.exe
+
+# run
+c_test_environment/join.exe INPUT_FILE.csv
+```
 
 ### Run the full MyriaL -> Grappa tests
 The default tests (just running `nosetests`) include tests for translation from MyriaL to Grappa code but do no checking of whether the Grappa program correctly executes the query. To actually run the Grappa queries: 
 
-1. get Grappa https://github.com/uwsampa/grappa and follow installation instructions in its BUILD.md
-2. set GRAPPA_HOME to root of Grappa
-3. set RACO_HOME to root of raco
-4. run tests:
+1. `export RACO_HOME=/path/to/raco`
+2. get Grappa https://github.com/uwsampa/grappa and follow installation instructions in its BUILD.md
+3. `export GRAPPA_HOME=/path/to/grappa`
+4. run tests: run this command from the `$RACO_HOME` directory
 ```bash
 PYTHONPATH=c_test_environment RACO_GRAPPA_TESTS=1 python -m unittest grappalang_myrial_tests.MyriaLGrappaTest
 ```
 
-### Visualize a Radish plan as a graph
-Pass the `-c` option to `scripts/myrial`.
+
+### Output Radish (Grappa) plan and source program
+```
+scripts/myrial -c examples/join.myl
+# the query implemented in Grappa is in join.cpp
+```
 
 # More Raco
-using Raco, manipulating plans, adding optimizer rules
+using Raco, manipulating plans, adding optimizer rules, developing Raco
 see [Raco in myria-docs](https://github.com/uwescience/myria-docs/blob/master/raco.markdown)
 
 # Authors and contact information

c_test_environment/clang_datalog_tests.py

@@ -3,14 +3,14 @@
 from testquery import ClangRunner
 from generate_test_relations import generate_default
 from generate_test_relations import need_generate
-import raco.language.clang as clang
-import raco.language.clangcommon as clangcommon
+import raco.backends.cpp as clang
+import raco.backends.cpp.cppcommon as cppcommon
 from raco.platform_tests import DatalogPlatformTest
 
 import sys
 sys.path.append('./examples')
-from emitcode import emitCode
 from osutils import Chdir
+from raco.cpp_datalog_utils import emitCode
 import os
 
 
@@ -24,7 +24,7 @@ def check(self, query, name):
     def check_file(self, query, name):
         with Chdir("c_test_environment") as d:
             os.remove("%s.cpp" % name) if os.path.exists("%s.cpp" % name) else None
-            emitCode(query, name, clang.CCAlgebra, emit_print=clangcommon.EMIT_FILE)
+            emitCode(query, name, clang.CCAlgebra, emit_print=cppcommon.EMIT_FILE)
             checkstore(name, ClangRunner())
 
     def setUp(self):

c_test_environment/clang_myrial_tests.py

@@ -3,7 +3,7 @@
 from testquery import ClangRunner
 from generate_test_relations import generate_default
 from generate_test_relations import need_generate
-from raco.language.clang import CCAlgebra
+from raco.backends.cpp import CCAlgebra
 from raco.platform_tests import MyriaLPlatformTestHarness, MyriaLPlatformTests
 from raco.compile import compile
 

c_test_environment/convert2bin.py

@@ -3,7 +3,7 @@
 import sys
 
 from raco.catalog import FromFileCatalog
-from raco.language.clangcommon import StagedTupleRef
+from raco.backends.cpp.cppcommon import StagedTupleRef
 
 """
 given a schema, creates a C++ program to convert csv data to a binary format

c_test_environment/grappa_ingest.py

@@ -13,8 +13,8 @@
 p = argparse.ArgumentParser(prog=sys.argv[0])
 p.add_argument("-i", dest="input_file", required=True, help="input file")
 p.add_argument("-c", dest="catalog_path", help="path of catalog file, see FromFileCatalog for format", required=True)
-p.add_argument("-s", dest="system", help="clang or grappa", default="clang")
 p.add_argument("-n", dest="relation_name", required=True, help="name of relation")
+p.add_argument("-s", dest="system", help="cpp or grappa", default="cpp")
 p.add_argument("--splits", dest="splits", action="store_true", help="input file is base directory of file splits (e.g. hdfs)")
 p.add_argument("--softlink-data", dest="softlink_data", action="store_true", help="data file softlinked rather than copied")
 p.add_argument("--local-softlink-data", dest="local_softlink_data", action="store_true", help="softlink locally, only use if --host!=localhost but want local softlink (e.g. NFS)")
@@ -182,4 +182,4 @@ def add_data_file(f):
 
 
 # example schema for this file:
-# public,adhoc,sp2bench,http://sampa-gw.cs.washington.edu:1337,922232,clang,3,binary,subject,predicate,object,STRING_TYPE,STRING_TYPE,STRING_TYPE
+# public,adhoc,sp2bench,http://sampa-gw.cs.washington.edu:1337,922232,cpp,3,binary,subject,predicate,object,STRING_TYPE,STRING_TYPE,STRING_TYPE

c_test_environment/grappalang_myrial_tests.py

@@ -3,8 +3,8 @@
 from testquery import GrappalangRunner
 from generate_test_relations import generate_default
 from generate_test_relations import need_generate
-from raco.language.grappalang import GrappaAlgebra
-import raco.language.grappalang as grappalang
+from raco.backends.radish import GrappaAlgebra
+import raco.backends.radish as grappalang
 from raco.platform_tests import MyriaLPlatformTestHarness, MyriaLPlatformTests
 from raco.compile import compile
 from nose.plugins.skip import SkipTest

c_test_environment/grappalang_tests.py

@@ -4,14 +4,14 @@
 import sys
 import unittest
 
-from raco.language.grappalang import GrappaAlgebra
+from raco.backends.radish import GrappaAlgebra
 
 from generate_test_relations import generate_default, need_generate
 from raco.platform_tests import DatalogPlatformTest
 from testquery import checkquery, checkstore, GrappalangRunner
 
 sys.path.append('./examples')
-from emitcode import emitCode
+from raco.cpp_datalog_utils import emitCode
 
 
 class DatalogGrappaTest(unittest.TestCase, DatalogPlatformTest):

c_test_environment/repr_myrial_tests.py

@@ -1,7 +1,7 @@
 import unittest
 from generate_test_relations import generate_default
 from generate_test_relations import need_generate
-from raco.language.myrialang import MyriaLeftDeepTreeAlgebra
+from raco.backends.myria import MyriaLeftDeepTreeAlgebra
 from raco.platform_tests import MyriaLPlatformTestHarness, MyriaLPlatformTests
 from raco.from_repr import plan_from_repr
 

c_test_environment/run_query.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-""" Executes clang runner and retrieves the results """
+""" Executes cpp runner and retrieves the results """
 
 import argparse
 import os
@@ -15,7 +15,7 @@ def parse_options(args):
     parser = argparse.ArgumentParser()
 
     parser.add_argument('platform', metavar='P', type=str,
-                        help='Type of platform to use: clang or grappa', choices=['grappa', 'clang'])
+                        help='Type of platform to use: cpp or grappa', choices=['grappa', 'cpp'])
 
     parser.add_argument('file', help='File containing platform source program')
     parser.add_argument('--query', help='File containing myrial query')
@@ -25,10 +25,10 @@ def parse_options(args):
     return ns
 
 
-from raco.language.clang import CCAlgebra
-from raco.language.grappalang import GrappaAlgebra
+from raco.backends.cpp import CCAlgebra
+from raco.backends.radish import GrappaAlgebra
 from raco.catalog import FromFileCatalog
-from raco.language.clangcommon import EMIT_FILE
+from raco.backends.cpp.cppcommon import EMIT_FILE
 from clang_processor import ClangProcessor
 
 
@@ -54,12 +54,12 @@ def main(args):
     if opt.platform == 'grappa':
         runner = GrappalangRunner()
         runner.run(name, abspath)
-    elif opt.platform == 'clang':
+    elif opt.platform == 'cpp':
         try:
             runner = ClangRunner()
             runner.run(name, abspath)
         except subprocess.CalledProcessError as e:
-            print 'clang runner for %s failed' % (name)
+            print 'cpp runner for %s failed' % (name)
             print e.output
             raise
 

examples/clog.py

@@ -1,5 +1,5 @@
-from emitcode import emitCode
-from raco.language import CCAlgebra
+from raco.cpp_datalog_utils import emitCode
+from raco.backends.cpp import CCAlgebra
 import sys
 
 import logging

examples/example_queries.py

@@ -1,5 +1,6 @@
-from raco.language import CCAlgebra, GrappaAlgebra
-from emitcode import emitCode
+from raco.backends.cpp import CCAlgebra
+from raco.backends.radish import GrappaAlgebra
+from raco.cpp_datalog_utils import emitCode
 import raco.rules as rules
 from raco.grappalang import GrappaShuffleHashJoin, GrappaSymmetricHashJoin, GrappaHashJoin
 

examples/grappalog.py

@@ -1,5 +1,5 @@
-from emitcode import emitCode
-from raco.language import GrappaAlgebra
+from raco.cpp_datalog_utils import emitCode
+from raco.backends.radish import GrappaAlgebra
 import sys
 
 import logging

raco/backends/__init__.py

@@ -0,0 +1,2 @@
+# everything in backend_common made public
+from backend_common import *

raco/backends/cpp/__init__.py

@@ -0,0 +1,2 @@
+# everything in cpp.py made public
+from cpp import *