.circleci/config.yml

@@ -63,7 +63,7 @@ test: &test
         name: Execute Pytest
         command: |
           docker-compose run -e PYTHONHASHSEED=$PYTHONHASHSEED -e CIRCLECI=$CIRCLECI ibis \
-            pytest ibis -m "not udf" \
+            pytest ibis -m "${PYTEST_MARK_EXPRESSION}" \
                         -ra \
                         --numprocesses auto \
                         --doctest-modules \
@@ -213,18 +213,21 @@ jobs:
     environment:
       - PYTHONHASHSEED: 0
       - PYTHON_VERSION: 3.5
+      - PYTEST_MARK_EXPRESSION: "not udf and not postgis"
 
   python36_test:
     <<: *test
     environment:
       - PYTHONHASHSEED: "random"
       - PYTHON_VERSION: 3.6
+      - PYTEST_MARK_EXPRESSION: "not udf"
 
   python37_test:
     <<: *test
     environment:
       - PYTHONHASHSEED: "random"
       - PYTHON_VERSION: 3.7
+      - PYTEST_MARK_EXPRESSION: "not udf"
 
   python36_conda_build:
     <<: *build

.isort.cfg

@@ -1,2 +1,2 @@
 [settings]
-known_third_party = asv,click,clickhouse_driver,dateutil,google,graphviz,impala,jinja2,kudu,multipledispatch,numpy,pandas,pkg_resources,plumbum,psycopg2,pyarrow,pydata_google_auth,pymapd,pymysql,pytest,pytz,regex,requests,ruamel,setuptools,sphinx_rtd_theme,sqlalchemy,toolz
+known_third_party = asv,click,clickhouse_driver,dateutil,google,graphviz,impala,jinja2,kudu,multipledispatch,numpy,pandas,pkg_resources,plumbum,psycopg2,pyarrow,pydata_google_auth,pygit2,pymapd,pymysql,pytest,pytz,regex,requests,ruamel,setuptools,sphinx_rtd_theme,sqlalchemy,toolz

azure-pipelines.yml

@@ -34,7 +34,7 @@ jobs:
     - script: conda install conda=$(conda.version)
       displayName: 'Install an appropriate conda version'
 
-    - script: conda create --name $(conda.env) python=$(python.version) numpy pandas pytables ruamel.yaml jinja2 pyarrow multipledispatch pymysql sqlalchemy psycopg2 graphviz click mock plumbum flake8
+    - script: conda create --name $(conda.env) python=$(python.version) numpy pandas pytables ruamel.yaml jinja2 pyarrow multipledispatch pymysql sqlalchemy psycopg2 graphviz click mock plumbum flake8 geopandas
       displayName: 'Create conda environment'
 
     - script: |
@@ -67,8 +67,8 @@ jobs:
     # - script: '"C:\\Program Files\\MariaDB 10.3\\bin\\mysql" -u root -e "GRANT ALL PRIVILEGES ON *.* TO ibis@localhost"'
     #   displayName: 'Setup privileges for ibis user in MySQL'
 
-    - script: choco install -y postgresql10 --params '/Password:postgres'
-      displayName: 'Install postgres from chocolatey'
+    #- script: choco install -y postgresql10 --params '/Password:postgres'
+      #displayName: 'Install postgres from chocolatey'
 
     - script: |
         call activate $(conda.env)
@@ -85,10 +85,10 @@ jobs:
     #     python ci/datamgr.py mysql
     #   displayName: 'Load MySQL data'
 
-    - script: |
-        call activate $(conda.env)
-        python ci/datamgr.py postgres --psql-path="C:/Program Files/PostgreSQL/10/bin/psql.exe"
-      displayName: 'Load PostgreSQL data'
+    #- script: |
+        #call activate $(conda.env)
+        #python ci/datamgr.py postgres --psql-path="C:/Program Files/PostgreSQL/10/bin/psql.exe" -t functional_alltypes -t diamonds -t batting -t awards_players
+      #displayName: 'Load PostgreSQL data'
 
     - script: |
         call activate $(conda.env)
@@ -102,7 +102,7 @@ jobs:
 
     - script: |
         call activate $(conda.env)
-        pytest --tb=short --junitxml="junit-$(python.version).xml" -n auto -m "not backend and not clickhouse and not impala and not hdfs and not bigquery and not mapd and not mysql" -ra ibis
+        pytest --tb=short --junitxml="junit-$(python.version).xml" -n auto -m "not backend and not clickhouse and not impala and not hdfs and not bigquery and not mapd and not mysql and not postgis and not postgresql" -ra ibis
       displayName: 'Run tests'
 
     # publish test results

ci/datamgr.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import sys
 import tempfile
 import warnings
 import zipfile
@@ -197,21 +198,55 @@ def parquet(tables, data_directory, ignore_missing_dependency, **params):
     type=click.File('rt'),
     default=str(SCRIPT_DIR / 'schema' / 'postgresql.sql'),
 )
-@click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
+@click.option('-t', '--tables', multiple=True, default=TEST_TABLES + ['geo'])
 @click.option('-d', '--data-directory', default=DATA_DIR)
-@click.option('-l', '--psql-path', type=click.Path(exists=True), default=None)
+@click.option(
+    '-l',
+    '--psql-path',
+    type=click.Path(exists=True),
+    required=os.name == 'nt',
+    default=None if os.name == 'nt' else '/usr/bin/psql',
+)
 def postgres(schema, tables, data_directory, psql_path, **params):
-    psql = local.get('psql', psql_path)
+    psql = local[psql_path]
     data_directory = Path(data_directory)
     logger.info('Initializing PostgreSQL...')
     engine = init_database(
         'postgresql', params, schema, isolation_level='AUTOCOMMIT'
     )
+    use_postgis = 'geo' in tables and sys.version_info >= (3, 6)
+    if use_postgis:
+        engine.execute("CREATE EXTENSION POSTGIS")
 
     query = "COPY {} FROM STDIN WITH (FORMAT CSV, HEADER TRUE, DELIMITER ',')"
     database = params['database']
     for table in tables:
         src = data_directory / '{}.csv'.format(table)
+
+        # If we are loading the geo sample data, handle the data types
+        # specifically so that PostGIS understands them as geometries.
+        if table == 'geo':
+            if not use_postgis:
+                continue
+            from geoalchemy2 import Geometry, WKTElement
+
+            srid = 4326
+            df = pd.read_csv(src)
+            df = df[df.columns[1:]].applymap(
+                lambda x: WKTElement(x, srid=srid)
+            )
+            df.to_sql(
+                'geo',
+                engine,
+                dtype={
+                    "geo_point": Geometry("POINT", srid=srid),
+                    "geo_linestring": Geometry("LINESTRING", srid=srid),
+                    "geo_polygon": Geometry("POLYGON", srid=srid),
+                    "geo_multipolygon": Geometry("MULTIPOLYGON", srid=srid),
+                },
+            )
+            continue
+
         load = psql[
             '--host',
             params['host'],

ci/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
 
   postgres:
-    image: postgres
+    image: mdillon/postgis
     hostname: postgres
     ports:
       - 5432:5432

ci/feedstock.py

@@ -30,17 +30,22 @@ def cli():
 
 default_repo = 'https://github.com/conda-forge/ibis-framework-feedstock'
 default_dest = os.path.join(tempfile.gettempdir(), 'ibis-framework-feedstock')
+default_branch = 'master'
 
 
 @cli.command()
 @click.argument('repo-uri', default=default_repo)
 @click.argument('destination', default=default_dest)
-def clone(repo_uri, destination):
-    if Path(destination).exists():
-        return
-
-    cmd = git['clone', repo_uri, destination]
-
+@click.option('-b', '--branch', default=default_branch)
+def clone(repo_uri, destination, branch):
+    if not Path(destination).exists():
+        cmd = git['clone', repo_uri, destination]
+        cmd(
+            stdout=click.get_binary_stream('stdout'),
+            stderr=click.get_binary_stream('stderr'),
+        )
+
+    cmd = git['-C', destination, 'checkout', branch]
     cmd(
         stdout=click.get_binary_stream('stdout'),
         stderr=click.get_binary_stream('stderr'),

ci/requirements-3.5-dev.yml

@@ -26,6 +26,7 @@ dependencies:
   - pre_commit
   - psycopg2
   - pyarrow<0.12
+  - pygit2
   - pymapd>=0.8.3,<0.11.0
   - pymysql
   - pytables>=3.0.0
@@ -46,6 +47,6 @@ dependencies:
   - pip:
     - seed-isort-config
     - pydata-google-auth
-    - pytest  # conda-forge only has <=3.8.1 for python 3.5
+    - pytest>=4.5  # conda-forge only has <=3.8.1 for python 3.5
     - pytest-cov
     - pytest-xdist

ci/requirements-3.6-dev.yml

@@ -8,6 +8,8 @@ dependencies:
   - clickhouse-sqlalchemy
   - cmake
   - flake8
+  - geoalchemy2
+  - geopandas
   - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.15.0
@@ -24,10 +26,11 @@ dependencies:
   - psycopg2
   - pyarrow>=0.12
   - pydata-google-auth
+  - pygit2
   - pymapd>=0.12.0
   - pymysql
   - pytables>=3.0.0
-  - pytest
+  - pytest>=4.5
   - pytest-cov
   - pytest-xdist
   - python=3.6

ci/requirements-3.7-dev.yml

@@ -8,6 +8,8 @@ dependencies:
   - clickhouse-sqlalchemy
   - cmake
   - flake8
+  - geoalchemy2
+  - geopandas
   - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.15.0
@@ -24,10 +26,11 @@ dependencies:
   - psycopg2
   - pyarrow>=0.12
   - pydata-google-auth
+  - pygit2
   - pymapd>=0.12.0
   - pymysql
   - pytables>=3.0.0
-  - pytest
+  - pytest>=4.5
   - pytest-cov
   - pytest-xdist
   - python=3.7

dev/genrelease.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+"""Generate release notes for Ibis."""
+
+import datetime
+import pathlib
+
+import click
+import pygit2
+import regex as re
+
+GITHUB_CLOSE_KEYWORDS = (
+    'close',
+    'closes',
+    'closed',
+    'fix',
+    'fixes',
+    'fixed',
+    'resolve',
+    'resolves',
+    'resolved',
+)
+
+KEYWORD_MAP = {
+    'bld': 'support',
+    'build': 'support',
+    'bug': 'bug',
+    'ci': 'support',
+    'doc': 'support',
+    'enhance': 'feature',
+    'enh': 'feature',
+    'feat': 'feature',
+    'feature': 'feature',
+    'supp': 'support',
+    'test': 'support',
+    'tst': 'support',
+    'rls': 'support',
+    'release': 'support',
+}
+
+
+def commits_between(repo, start_ref, end_ref, options=None):
+    """Yield commits from `repo` between `start_ref` and `end_ref`."""
+    start = repo.revparse_single(start_ref)
+    end = repo.revparse_single(end_ref)
+    for commit in repo.walk(end.oid, options):
+        if commit.oid == start.oid:
+            return
+        else:
+            yield commit
+
+
+def iter_release_notes(repo, from_ref, to_ref, default_role):
+    """Yield release notes from `from_ref` to `to_ref`."""
+    pattern = re.compile(
+        r'^(?:{})\s+#(\d+)\s+from'.format('|'.join(GITHUB_CLOSE_KEYWORDS)),
+        flags=re.MULTILINE | re.IGNORECASE,
+    )
+    for commit in commits_between(
+        repo, from_ref, to_ref, options=pygit2.GIT_SORT_TOPOLOGICAL
+    ):
+        message = commit.message.strip()
+        subject, *lines = map(str.strip, message.splitlines())
+        tag, *rest = subject.split(':', 1)
+        tag = tag.lower()
+        lineitem = ''.join(rest) or subject
+        role = KEYWORD_MAP.get(tag, default_role)
+        modifier = ' major' if role == 'bug' else ''
+        try:
+            issue_number, *_ = pattern.findall(message)
+        except ValueError:
+            issue_number = '-'
+        yield "* :{role}:`{issue_number}{modifier}` {lineitem}".format(
+            role=role,
+            issue_number=issue_number,
+            modifier=modifier,
+            lineitem=lineitem.strip(),
+        )
+
+
+repo_path = pathlib.Path(__file__).parent.parent
+repo = pygit2.Repository(str(repo_path))
+
+
+@click.command()
+@click.argument('release_version')
+@click.option(
+    '-f',
+    '--from',
+    'from_',
+    default=repo.describe(
+        describe_strategy=pygit2.GIT_DESCRIBE_TAGS, abbreviated_size=0
+    ),
+    help=(
+        "The reference from which to calculate release notes. Defaults to "
+        "the most recent tag."
+    ),
+    show_default=True,
+)
+@click.option(
+    '-t',
+    '--to',
+    default="upstream/master",
+    help="The last reference to include in release notes.",
+    show_default=True,
+)
+@click.option(
+    '-d',
+    '--release-date',
+    type=str,
+    default=datetime.datetime.now().date().strftime('%Y-%m-%d'),
+    help="The date of the release. Defaults to the current date.",
+    show_default=True,
+)
+@click.option(
+    '-r',
+    '--default-role',
+    default='support',
+    help=(
+        "The Sphinx role to use if a known prefix is not found in a "
+        "commit's subject line."
+    ),
+    show_default=True,
+)
+def main(release_version, from_, to, release_date, default_role):
+    title = "* :release:`{release} <{date}>`".format(
+        release=release_version, date=release_date
+    )
+    click.echo(title)
+    click.echo('\n'.join(iter_release_notes(repo, from_, to, default_role)))
+
+
+if __name__ == '__main__':
+    main()