Added ci; docs; tests;

ci/datamgr.py

@@ -7,6 +7,7 @@
 
 import pandas as pd
 import sqlalchemy as sa
+import pymapd
 
 from toolz import dissoc
 from plumbum import local
@@ -188,6 +189,44 @@ def sqlite(database, schema, tables, data_directory, **params):
     insert_tables(engine, tables, data_directory)
 
 
+@cli.command()
+@click.option('-h', '--host', default='localhost')
+@click.option('-P', '--port', default=9091, type=int)
+@click.option('-u', '--user', default='mapd')
+@click.option('-p', '--password', default='HyperInteractive')
+@click.option('-D', '--database', default='mapd')
+@click.option('-S', '--schema', type=click.File('rt'),
+              default=str(SCRIPT_DIR / 'schema' / 'mapd.sql'))
+@click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
+@click.option('-d', '--data-directory', default=DATA_DIR)
+def mapd(schema, tables, data_directory, **params):
+    data_directory = Path(data_directory)
+    click.echo('Initializing MapD...')
+
+    # connection
+    conn = pymapd.connect(
+        host=params['host'], user=params['user'],
+        password=params['password'],
+        port=params['port'], dbname=params['database']
+    )
+
+    # create database
+    for stmt in schema.read().split(';'):
+        stmt = stmt.strip()
+        if len(stmt):
+            conn.execute(stmt)
+
+    # import data
+    query = 'COPY {} FROM \'{}\' WITH(delimiter=\',\', header=\'true\')'
+
+    click.echo('Loading data ...')
+    for table in tables:
+        src = data_directory / '{}.csv'.format(table)
+        click.echo(src)
+        conn.execute(query.format(table, src))
+    conn.close()
+
+
 @cli.command()
 @click.option('-h', '--host', default='localhost')
 @click.option('-P', '--port', default=3306, type=int)

ci/docker-compose.yml

@@ -8,6 +8,16 @@ services:
     environment:
       POSTGRES_PASSWORD: postgres
 
+  mapd:
+    image: mapd/mapd-ce-cpu
+    ports:
+      - 9090-9092:9090-9092
+    environment:
+      - MAPD_HOST=mapd
+      - MAPD_PORT=9091
+      - MAPD_DATABASE=mapd
+      - MAPD_USER=mapd
+
   mysql:
     image: mariadb:10.2
     ports:
@@ -52,7 +62,8 @@ services:
   waiter:
     image: jwilder/dockerize
     command: |
-      dockerize -wait tcp://mysql:3306
+      dockerize -wait tcp://mapd:9091
+                -wait tcp://mysql:3306
                 -wait tcp://postgres:5432
                 -wait tcp://impala:21050
                 -wait tcp://impala:50070
@@ -84,6 +95,11 @@ services:
       - IBIS_TEST_CLICKHOUSE_HOST=clickhouse
       - IBIS_TEST_CLICKHOUSE_PORT=9000
       - IBIS_TEST_CLICKHOUSE_DATABASE=ibis_testing
+      - IBIS_TEST_MAPD_HOST=mapd
+      - IBIS_TEST_MAPD_PORT=9091
+      - IBIS_TEST_MAPD_DATABASE=mapd
+      - IBIS_TEST_MAPD_USER=mapd
+      - IBIS_TEST_MAPD_PASSWORD=HyperInteractive
       - GOOGLE_BIGQUERY_PROJECT_ID=ibis-gbq
       - GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-service-key.json
     volumes:

ci/requirements-dev-2.7.yml

@@ -23,6 +23,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/requirements-dev-3.5.yml

@@ -18,6 +18,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytest
   - python=3.5

ci/requirements-dev-3.6.yml

@@ -18,6 +18,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/requirements-docs-3.6.yml

@@ -22,6 +22,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/schema/mapd.sql

@@ -0,0 +1,72 @@
+DROP TABLE IF EXISTS diamonds;
+
+CREATE TABLE diamonds (
+    carat FLOAT,
+    cut TEXT,
+    color TEXT,
+    clarity TEXT,
+    depth FLOAT,
+    table_ FLOAT,
+    price BIGINT,
+    x FLOAT,
+    y FLOAT,
+    z FLOAT
+);
+
+DROP TABLE IF EXISTS batting;
+
+CREATE TABLE batting (
+    playerID VARCHAR(255),
+    yearID BIGINT,
+    stint BIGINT,
+    teamID VARCHAR(7),
+    lgID VARCHAR(7),
+    G BIGINT,
+    AB BIGINT,
+    R BIGINT,
+    H BIGINT,
+    X2B BIGINT,
+    X3B BIGINT,
+    HR BIGINT,
+    RBI BIGINT,
+    SB BIGINT,
+    CS BIGINT,
+    BB BIGINT,
+    SO BIGINT,
+    IBB BIGINT,
+    HBP BIGINT,
+    SH BIGINT,
+    SF BIGINT,
+    GIDP BIGINT
+);
+
+DROP TABLE IF EXISTS awards_players;
+
+CREATE TABLE awards_players (
+    playerID VARCHAR(255),
+    awardID VARCHAR(255),
+    yearID BIGINT,
+    lgID VARCHAR(7),
+    tie VARCHAR(7),
+    notes VARCHAR(255)
+);
+
+DROP TABLE IF EXISTS functional_alltypes;
+
+CREATE TABLE functional_alltypes (
+    index BIGINT,
+    Unnamed_ BIGINT,
+    id INTEGER,
+    bool_col BOOLEAN,
+    tinyint_col SMALLINT,
+    smallint_col SMALLINT,
+    int_col INTEGER,
+    bigint_col BIGINT,
+    float_col FLOAT,
+    double_col DOUBLE,
+    date_string_col TEXT,
+    string_col TEXT,
+    timestamp_col TIMESTAMP,
+    year_ INTEGER,
+    month_ INTEGER
+);

docs/source/developer.rst

@@ -139,6 +139,25 @@ instructions above, then SQLite will be available in the conda environment.
 
       ci/datamgr.py sqlite
 
+MapD
+^^^^
+
+MapD can be used from either a docker image or from your machine directly.
+
+#. **Start the MapD Server docker image in another terminal**:
+
+   .. code:: sh
+
+      # Keeping this running as long as you want to test ibis
+      docker run -d -v $HOME/mapd-docker-storage:/mapd-storage -p 9090-9092:9090-9092 mapd/mapd-ce-cpu
+
+
+Here's how to load test data into MapD:
+
+   .. code:: sh
+
+      ci/datamgr.py mapd
+
 
 Running Tests
 -------------

ibis/expr/api.py

@@ -76,7 +76,7 @@
     'negate', 'ifelse',
     'Expr', 'Schema',
     'window', 'trailing_window', 'cumulative_window',
-    'pi'
+    'pi', 'distance'
 ]
 
 
@@ -3200,3 +3200,11 @@ def _table_drop(self, fields):
 
 
 _add_methods(ir.TableExpr, _table_methods)
+
+
+# geometric operation
+
+def distance(from_lon, from_lat, to_lon, to_lat):
+    """
+    """
+    return ops.Distance(from_lon, from_lat, to_lon, to_lat).to_expr()

ibis/expr/operations.py

@@ -2782,3 +2782,17 @@ def root_tables(self):
     def _make_expr(self):
         dtype = rlz.highest_precedence_dtype(self.values)
         return ir.ListExpr(self, dtype=dtype)
+
+
+# GEOMETRIC OPERATIONS
+
+class Distance(ValueOp):
+    """
+    Calculates distance in meters between two WGS-84 positions.
+
+    """
+    from_lon = Arg(rlz.column(rlz.numeric))
+    from_lat = Arg(rlz.column(rlz.numeric))
+    to_lon = Arg(rlz.column(rlz.numeric))
+    to_lat = Arg(rlz.column(rlz.numeric))
+    output_type = rlz.shape_like('from_lon', dt.float)

ibis/mapd/api.py

@@ -1,6 +1,6 @@
 from ibis.config import options
 from ibis.mapd.compiler import dialect, compiles, rewrites
-from ibis.mapd.client import MapDClient
+from ibis.mapd.client import MapDClient, EXECUTION_TYPE_CURSOR
 
 import ibis.common as com
 
@@ -31,9 +31,9 @@ def verify(expr, params=None):
 
 
 def connect(
-    uri: str=None, user: str=None, password: str=None, host: str=None,
-    port: int=9091, dbname: str=None, protocol: str='binary',
-    execution_type: int=3
+    uri=None, user=None, password=None, host=None,
+    port=9091, database=None, protocol='binary',
+    execution_type=EXECUTION_TYPE_CURSOR
 ):
     """Create a MapDClient for use with Ibis
 
@@ -44,7 +44,7 @@ def connect(
     :param password: str
     :param host: str
     :param port: int
-    :param dbname: str
+    :param database: str
     :param protocol: str
     :param execution_type: int
     Returns
@@ -54,7 +54,7 @@ def connect(
     """
     client = MapDClient(
         uri=uri, user=user, password=password, host=host,
-        port=port, dbname=dbname, protocol=protocol,
+        port=port, database=database, protocol=protocol,
         execution_type=execution_type
     )
 

ibis/mapd/client.py

@@ -34,6 +34,7 @@ class MapDDataType(object):
     dtypes = {
         'BIGINT': dt.int64,
         'BOOLEAN': dt.Boolean,
+        'BOOL': dt.Boolean,
         'CHAR': dt.string,
         'DATE': dt.date,
         'DECIMAL': dt.float64,
@@ -140,9 +141,9 @@ class MapDClient(SQLClient):
     dialect = MapDDialect
 
     def __init__(
-        self, uri: str=None, user: str=None, password: str=None,
-        host: str=None, port: int=9091, dbname: str=None,
-        protocol: str='binary', execution_type: int=3
+        self, uri=None, user=None, password=None,
+        host=None, port=9091, database=None,
+        protocol='binary', execution_type=EXECUTION_TYPE_CURSOR
     ):
         """
 
@@ -153,17 +154,19 @@ def __init__(
         password : str
         host : str
         port : int
-        dbname : str
+        database : str
         protocol : {‘binary’, ‘http’, ‘https’}
-        execution_type : {1, 2, 3}
+        execution_type : {
+          EXECUTION_TYPE_ICP, EXECUTION_TYPE_ICP_GPU, EXECUTION_TYPE_CURSOR
+        }
 
         """
         self.uri = uri
         self.user = user
         self.password = password
         self.host = host
         self.port = port
-        self.dbname = dbname
+        self.db_name = database
         self.protocol = protocol
 
         if execution_type not in (
@@ -177,7 +180,7 @@ def __init__(
 
         self.con = pymapd.connect(
             uri=uri, user=user, password=password, host=host,
-            port=port, dbname=dbname, protocol=protocol
+            port=port, dbname=database, protocol=protocol
         )
 
     def log(self, msg):
@@ -259,14 +262,14 @@ def database(self, name=None):
             client_class = type(self)
             new_client = client_class(
                 uri=self.uri, user=self.user, password=self.password,
-                host=self.host, port=self.port, dbname=name,
+                host=self.host, port=self.port, database=name,
                 protocol=self.protocol, execution_type=self.execution_type
             )
             return self.database_class(name, new_client)
 
     @property
     def current_database(self):
-        return self.dbname
+        return self.db_name
 
     def set_database(self, name):
         raise NotImplementedError(
@@ -280,7 +283,7 @@ def exists_database(self, name):
     def list_databases(self, like=None):
         raise NotImplementedError()
 
-    def exists_table(self, name: str, database: str=None):
+    def exists_table(self, name, database=None):
         """
         Determine if the indicated table or view exists
 
@@ -296,7 +299,11 @@ def exists_table(self, name: str, database: str=None):
         return bool(self.list_tables(like=name, database=database))
 
     def list_tables(self, like=None, database=None):
-        return self.con.get_tables()
+        tables = self.con.get_tables()
+
+        if like is None:
+            return tables
+        return list(filter(lambda t: t == like, tables))
 
     def get_schema(self, table_name, database=None):
         """