Clickhouse query backend

.circleci/config.yml

@@ -4,6 +4,10 @@ environment: &environment
   - IBIS_TEST_POSTGRES_DB: ibis_testing
   - IBIS_POSTGRES_USER: ubuntu
   - IBIS_POSTGRES_PASS: ubuntu
+  - IBIS_TEST_CLICKHOUSE_DB: ibis_testing
+  - IBIS_CLICKHOUSE_USER: default
+  - IBIS_CLICKHOUSE_HOST: localhost
+  - IBIS_CLICKHOUSE_PASS: ''
 
   - DATA_URL: https://storage.googleapis.com/ibis-ci-data
 
@@ -49,6 +53,13 @@ python_test_steps: &python_test_steps
         --script ci/postgresql_load.sql \
         functional_alltypes diamonds awards_players batting
 
+  - run: |
+      ci/datamgr.py clickhouse \
+        --database "$IBIS_TEST_POSTGRES_DB" \
+        --data-directory "$DATA_DIR" \
+        --script ci/clickhouse_load.sql \
+        functional_alltypes diamonds awards_players batting
+
   - run: test_data_admin.py load --data --data-dir "$DATA_DIR"
   - run: mkdir -p /tmp/reports
   - run: |

ci/clickhouse_load.sql

@@ -0,0 +1,96 @@
+DROP TABLE IF EXISTS diamonds;
+
+CREATE TABLE diamonds (
+    `date` Date DEFAULT today(),
+    carat Float64,
+    cut String,
+    color String,
+    clarity String,
+    depth Float64,
+    `table` Float64,
+    price Int64,
+    x Float64,
+    y Float64,
+    z Float64
+) ENGINE = MergeTree(date, (`carat`), 8192);
+
+DROP TABLE IF EXISTS batting;
+
+CREATE TABLE batting (
+    `date` Date DEFAULT today(),
+    `playerID` String,
+    `yearID` Int64,
+    stint Int64,
+    `teamID` String,
+    `lgID` String,
+    `G` Int64,
+    `AB` Int64,
+    `R` Int64,
+    `H` Int64,
+    `X2B` Int64,
+    `X3B` Int64,
+    `HR` Int64,
+    `RBI` Int64,
+    `SB` Int64,
+    `CS` Int64,
+    `BB` Int64,
+    `SO` Int64,
+    `IBB` Int64,
+    `HBP` Int64,
+    `SH` Int64,
+    `SF` Int64,
+    `GIDP` Int64
+) ENGINE = MergeTree(date, (`playerID`), 8192);
+
+DROP TABLE IF EXISTS awards_players;
+
+CREATE TABLE awards_players (
+    `date` Date DEFAULT today(),
+    `playerID` String,
+    `awardID` String,
+    `yearID` Int64,
+    `lgID` String,
+    tie String,
+    notes String
+) ENGINE = MergeTree(date, (`playerID`), 8192);
+
+DROP TABLE IF EXISTS functional_alltypes;
+
+CREATE TABLE functional_alltypes (
+    `date` Date DEFAULT toDate(timestamp_col),
+    `index` Int64,
+    `Unnamed_0` Int64,
+    id Int32,
+    bool_col UInt8,
+    tinyint_col Int8,
+    smallint_col Int16,
+    int_col Int32,
+    bigint_col Int64,
+    float_col Float32,
+    double_col Float64,
+    date_string_col String,
+    string_col String,
+    timestamp_col DateTime,
+    year Int32,
+    month Int32
+) ENGINE = MergeTree(date, (`index`), 8192);
+
+DROP TABLE IF EXISTS tzone;
+
+CREATE TABLE tzone (
+    `date` Date DEFAULT today(),
+    ts DateTime,
+    key String,
+    value Float64
+) ENGINE = MergeTree(date, (key), 8192);
+
+DROP TABLE IF EXISTS array_types;
+
+CREATE TABLE IF NOT EXISTS array_types (
+    `date` Date DEFAULT today(),
+    x Array(Int64),
+    y Array(String),
+    z Array(Float64),
+    grouper String,
+    scalar_column Float64
+) ENGINE = MergeTree(date, (scalar_column), 8192);

ci/datamgr.py

@@ -24,6 +24,83 @@ def cli():
     pass
 
 
+@cli.command()
+@click.argument('tables', nargs=-1)
+@click.option('-S', '--script', type=click.File('rt'), required=True)
+@click.option(
+    '-d', '--database',
+    default=os.environ.get('IBIS_TEST_CLICKHOUSE_DB', 'ibis_testing')
+)
+@click.option(
+    '-D', '--data-directory',
+    default=tempfile.gettempdir(), type=click.Path(exists=True)
+)
+def clickhouse(script, tables, database, data_directory):
+    username = os.environ.get('IBIS_CLICKHOUSE_USER', 'default')
+    host = os.environ.get('IBIS_CLICKHOUSE_HOST', 'localhost')
+    password = os.environ.get('IBIS_CLICKHOUSE_PASS', '')
+
+    url = sa.engine.url.URL(
+        'clickhouse+native',
+        username=username,
+        host=host,
+        password=password,
+    )
+    engine = sa.create_engine(str(url))
+    engine.execute('DROP DATABASE IF EXISTS "{}"'.format(database))
+    engine.execute('CREATE DATABASE "{}"'.format(database))
+
+    url = sa.engine.url.URL(
+        'clickhouse+native',
+        username=username,
+        host=host,
+        password=password,
+        database=database,
+    )
+    engine = sa.create_engine(str(url))
+    script_text = script.read()
+
+    # missing stmt
+    # INSERT INTO array_types (x, y, z, grouper, scalar_column) VALUES
+    # ([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0),
+    # ([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0),
+    # ([6], ['f'], [6.0], 'a', 3.0),
+    # ([1], ['a'], [], 'b', 4.0),
+    # ([2, 3], ['b', 'c'], [], 'b', 5.0),
+    # ([4, 5], ['d', 'e'], [4.0, 5.0], 'c', 6.0);
+
+    with engine.begin() as con:
+        # doesn't support multiple statements
+        for stmt in script_text.split(';'):
+            if len(stmt.strip()):
+                con.execute(stmt)
+
+    table_paths = [
+        os.path.join(data_directory, '{}.csv'.format(table))
+        for table in tables
+    ]
+    dtype = {'bool_col': np.bool_}
+    for table, path in zip(tables, table_paths):
+        # correct dtypes per table to be able to insert
+        # TODO: cleanup, kinda ugly
+        df = pd.read_csv(path, index_col=None, header=0, dtype=dtype)
+        if table == 'functional_alltypes':
+            df = df.rename(columns={'Unnamed: 0': 'Unnamed_0'})
+            cols = ['date_string_col', 'string_col']
+            df[cols] = df[cols].astype(str)
+            df.timestamp_col = df.timestamp_col.astype('datetime64[s]')
+        elif table == 'batting':
+            cols = ['playerID', 'teamID', 'lgID']
+            df[cols] = df[cols].astype(str)
+            cols = df.select_dtypes([float]).columns
+            df[cols] = df[cols].fillna(0).astype(int)
+        elif table == 'awards_players':
+            cols = ['playerID', 'awardID', 'lgID', 'tie', 'notes']
+            df[cols] = df[cols].astype(str)
+
+        df.to_sql(table, engine, index=False, if_exists='append')
+
+
 @cli.command()
 @click.argument('tables', nargs=-1)
 @click.option('-S', '--script', type=click.File('rt'), required=True)
@@ -103,6 +180,8 @@ def sqlite(script, tables, database, data_directory):
         os.path.join(data_directory, '{}.csv'.format(table))
         for table in tables
     ]
+    click.echo(tables)
+    click.echo(table_paths)
     for table, path in zip(tables, table_paths):
         df = pd.read_csv(path, index_col=None, header=0)
         with engine.begin() as con:

ci/requirements-dev-2.7.yml

@@ -23,3 +23,5 @@ dependencies:
   - toolz
   - pip:
     - hdfs>=2.0.0
+    - clickhouse-driver
+    - clickhouse-sqlalchemy

ci/requirements-dev-3.4.yml

@@ -21,3 +21,5 @@ dependencies:
   - toolz
   - pip:
     - hdfs>=2.0.0
+    - clickhouse-driver
+    - clickhouse-sqlalchemy

ci/requirements-dev-3.5.yml

@@ -21,3 +21,5 @@ dependencies:
   - toolz
   - pip:
     - hdfs>=2.0.0
+    - clickhouse-driver
+    - clickhouse-sqlalchemy

ci/requirements-dev-3.6.yml

@@ -21,3 +21,5 @@ dependencies:
   - toolz
   - pip:
     - hdfs>=2.0.0
+    - clickhouse-driver
+    - clickhouse-sqlalchemy

docs/source/developer.rst

@@ -69,6 +69,26 @@ Impala (with UDFs)
 
       test_data_admin.py load --data --data-dir=$DATA_DIR
 
+Clickhouse
+^^^^^^^^^^
+
+#. **Start the Clickhouse Server docker image in another terminal**:
+
+   .. code:: sh
+
+      # Keeping this running as long as you want to test ibis
+      docker run --rm -p 9000:9000 --tty yandex/clickhouse-server
+
+#. **Load data**:
+
+   .. code:: sh
+
+      ci/datamgr.py clickhouse \
+          --database $IBIS_TEST_CLICKHOUSE_DB \
+          --data-directory $DATA_DIR/ibis-testing-data \
+          --script ci/clickhouse_load.sql \
+          functional_alltypes batting diamonds awards_players
+
 PostgreSQL
 ^^^^^^^^^^
 
@@ -81,7 +101,7 @@ Here's how to load test data into PostgreSQL:
 
       ci/datamgr.py postgres \
           --database $IBIS_TEST_POSTGRES_DB \
-          --data-directory $DATA_DIR \
+          --data-directory $DATA_DIR/ibis-testing-data \
           --script ci/postgresql_load.sql \
           functional_alltypes batting diamonds awards_players
 
@@ -95,7 +115,7 @@ instructions above, then SQLite will be available in the conda environment.
 
       ci/datamgr.py sqlite \
           --database $IBIS_TEST_SQLITE_DB_PATH \
-          --data-directory $DATA_DIR \
+          --data-directory $DATA_DIR/ibis-testing-data \
           --script ci/sqlite_load.sql \
           functional_alltypes batting diamonds awards_players
 

ibis/__init__.py

@@ -39,6 +39,11 @@
 except ImportError:  # pip install ibis-framework[postgres]
     pass
 
+try:
+    import ibis.clickhouse.api as clickhouse
+except ImportError:  # pip install ibis-framework[clickhouse]
+    pass
+
 try:
     from multipledispatch import halt_ordering, restart_ordering
     halt_ordering()

ibis/clickhouse/api.py

@@ -0,0 +1,81 @@
+import ibis.common as com
+
+from ibis.config import options
+from ibis.clickhouse.client import ClickhouseClient
+
+
+def compile(expr):
+    """
+    Force compilation of expression as though it were an expression depending
+    on Clickhouse. Note you can also call expr.compile()
+
+    Returns
+    -------
+    compiled : string
+    """
+    from .compiler import to_sql
+    return to_sql(expr)
+
+
+def verify(expr):
+    """
+    Determine if expression can be successfully translated to execute on
+    Clickhouse
+    """
+    try:
+        compile(expr)
+        return True
+    except com.TranslationError:
+        return False
+
+
+def connect(host='localhost', port=9000, database='default', user='default',
+            password='', client_name='ibis', compression=False):
+    """Create an ClickhouseClient for use with Ibis.
+
+    Parameters
+    ----------
+    host : str, optional
+        Host name of the clickhouse server
+    port : int, optional
+        Clickhouse server's  port
+    database : str, optional
+        Default database when executing queries
+    user : str, optional
+        User to authenticate with
+    password : str, optional
+        Password to authenticate with
+    client_name: str, optional
+        This will appear in clickhouse server logs
+    compression: str, optional
+        Weather or not to use compression. Default is False.
+        Possible choices: lz4, lz4hc, quicklz, zstd
+        True is equivalent to 'lz4'.
+
+    Examples
+    --------
+    >>> import ibis
+    >>> import os
+    >>> clickhouse_host = os.environ.get('IBIS_TEST_CLICKHOUSE_HOST',
+    ...                                  'localhost')
+    >>> clickhouse_port = int(os.environ.get('IBIS_TEST_CLICKHOUSE_PORT',
+    ...                                      9000))
+    >>> client = ibis.clickhouse.connect(
+    ...     host=clickhouse_host,
+    ...     port=clickhouse_port
+    ... )
+    >>> client  # doctest: +ELLIPSIS
+    <ibis.clickhouse.client.ClickhouseClient object at 0x...>
+
+    Returns
+    -------
+    ClickhouseClient
+    """
+
+    client = ClickhouseClient(host, port=port, database=database, user=user,
+                              password=password, client_name=client_name,
+                              compression=compression)
+    if options.default_backend is None:
+        options.default_backend = client
+
+    return client