docs(backends): document memtable support and performance for each ba…

…ckend
ibis-project · Aug 1, 2023 · b321733 · b321733
1 parent 8966b5d
commit b321733
Show file tree

Hide file tree

Showing 20 changed files with 66 additions and 0 deletions.
diff --git a/docs/backends/bigquery.md b/docs/backends/bigquery.md
@@ -2,12 +2,15 @@
 backend_name: Google BigQuery
 backend_url: https://cloud.google.com/bigquery
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # BigQuery
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the BigQuery backend:

diff --git a/docs/backends/clickhouse.md b/docs/backends/clickhouse.md
@@ -3,12 +3,15 @@ backend_name: ClickHouse
 backend_url: https://clickhouse.yandex/
 backend_module: clickhouse
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: native
 ---
 
 # ClickHouse
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the ClickHouse backend:

diff --git a/docs/backends/dask.md b/docs/backends/dask.md
@@ -3,6 +3,7 @@ backend_name: Dask
 backend_url: https://dask.org
 backend_module: dask
 backend_param_style: a dictionary of paths
+memtable_impl: native
 ---
 
 {% include 'backends/template.md' %}
diff --git a/docs/backends/datafusion.md b/docs/backends/datafusion.md
@@ -5,12 +5,15 @@ backend_module: datafusion
 version_added: "2.1"
 exports: ["PyArrow", "Parquet", "Delta Lake", "CSV", "Pandas"]
 imports: ["CSV", "Parquet", "Delta Lake"]
+memtable_impl: not_implemented
 ---
 
 # DataFusion
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Apache Datafusion backend:

diff --git a/docs/backends/druid.md b/docs/backends/druid.md
@@ -3,6 +3,7 @@ backend_name: Druid
 backend_url: https://druid.apache.org/
 backend_module: druid
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # Druid
@@ -13,6 +14,8 @@ exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
 
     The Druid backend is experimental and is subject to backwards incompatible changes.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Druid backend:

diff --git a/docs/backends/duckdb.md b/docs/backends/duckdb.md
@@ -14,6 +14,7 @@ imports:
     "SQLite",
     "Postgres",
   ]
+memtable_impl: native
 ---
 
 # DuckDB
@@ -28,6 +29,8 @@ imports:
       See [this issue](https://github.com/ibis-project/ibis/issues/4503) for
       more details.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the DuckDB backend:

diff --git a/docs/backends/impala.md b/docs/backends/impala.md
@@ -8,6 +8,7 @@ intro: |
   without requiring you to switch back and forth between Python code and the
   Impala shell.
 exclude_backend_api: true
+memtable_impl: fallback
 ---
 
 {% include 'backends/template.md' %}

diff --git a/docs/backends/memtable-template.md b/docs/backends/memtable-template.md
@@ -0,0 +1,18 @@
+{% set memtable_badges = {"native": "56ae57", "fallback": "goldenrod", "none": "ff6961"} %}
+
+## `ibis.memtable` Support ![memtable](https://img.shields.io/badge/{{ memtable_impl }}-{{ memtable_badges[memtable_impl] }})
+
+{% if memtable_impl == "not_implemented" %}
+
+The {{ backend_name }} does not currently support in-memory tables.
+
+Please [file an issue](https://github.com/ibis-project/ibis/issues/new/choose)
+if you'd like the {{ backend_name }} to support in-memory tables.
+
+{% else %}
+
+The {{ backend_name }} backend supports `memtable`s {% if memtable_impl == "fallback" %} by constructing a string with the contents of the in-memory object. **This will be very inefficient for medium to large in-memory tables**. Please [file an issue](https://github.com/ibis-project/ibis/issues/new/choose) if you observe performance issues when using in-memory tables. {% elif memtable_impl == "native" %} by natively executing queries against the underlying storage (e.g., pyarrow Tables or pandas DataFrames).
+
+{% endif %}
+
+{% endif %}
diff --git a/docs/backends/mssql.md b/docs/backends/mssql.md
@@ -5,12 +5,15 @@ backend_module: mssql
 backend_param_style: connection parameters
 version_added: "4.0"
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # MSSQL
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the MSSQL backend:

diff --git a/docs/backends/mysql.md b/docs/backends/mysql.md
@@ -4,12 +4,15 @@ backend_url: https://www.mysql.com/
 backend_module: mysql
 backend_param_style: a SQLAlchemy-style URI
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # MySQL
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the MySQL backend:

diff --git a/docs/backends/oracle.md b/docs/backends/oracle.md
@@ -7,6 +7,7 @@ backend_connection_example: ibis.connect("oracle://user:pass@host:port/service_n
 is_experimental: true
 version_added: "6.0"
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # Oracle
@@ -17,6 +18,8 @@ exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
 
     The Oracle backend is experimental and is subject to backwards incompatible changes.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Oracle backend:

diff --git a/docs/backends/pandas.md b/docs/backends/pandas.md
@@ -6,6 +6,7 @@ intro: Ibis's pandas backend is available in core Ibis.
 backend_param_style: a dictionary of paths
 do_connect_base: BasePandasBackend
 is_core: true
+memtable_impl: native
 ---
 
 {% include 'backends/template.md' %}

diff --git a/docs/backends/polars.md b/docs/backends/polars.md
@@ -6,6 +6,7 @@ is_experimental: true
 version_added: "4.0"
 exports: ["PyArrow", "Parquet", "Delta Lake", "CSV", "Pandas"]
 imports: ["CSV", "Parquet", "Delta Lake", "Pandas"]
+memtable_impl: native
 ---
 
 # Polars
@@ -16,6 +17,8 @@ imports: ["CSV", "Parquet", "Delta Lake", "Pandas"]
 
     The Polars backend is experimental and is subject to backwards incompatible changes.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Polars backend:

diff --git a/docs/backends/postgresql.md b/docs/backends/postgresql.md
@@ -4,12 +4,15 @@ backend_url: https://www.postgresql.org/
 backend_module: postgres
 backend_param_style: a SQLAlchemy-style URI
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: native
 ---
 
 # PostgreSQL
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Postgres backend:

diff --git a/docs/backends/pyspark.md b/docs/backends/pyspark.md
@@ -5,12 +5,15 @@ backend_module: pyspark
 backend_param_style: PySpark things
 exports: ["PyArrow", "Parquet", "Delta Lake", "Pandas"]
 imports: ["CSV", "Parquet", "Delta Lake"]
+memtable_impl: native
 ---
 
 # PySpark
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the PySpark backend:

diff --git a/docs/backends/snowflake.md b/docs/backends/snowflake.md
@@ -2,6 +2,7 @@
 backend_name: Snowflake
 backend_url: https://snowflake.com/
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: native
 ---
 
 # Snowflake
@@ -12,6 +13,8 @@ exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
 
     The Snowflake backend is experimental and is subject to backwards incompatible changes.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Snowflake backend:

diff --git a/docs/backends/sqlite.md b/docs/backends/sqlite.md
@@ -3,12 +3,15 @@ backend_name: SQLite
 backend_url: https://www.sqlite.org/
 backend_module: sqlite
 imports: ["CSV", "Parquet", "JSON", "PyArrow", "Pandas", "SQLite", "Postgres"]
+memtable_impl: fallback
 ---
 
 # SQLite
 
 {% include 'backends/badges.md' %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the SQLite backend:

diff --git a/docs/backends/template.md b/docs/backends/template.md
@@ -13,6 +13,8 @@
 
 {% if not (development_only | default(False)) %}
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the {{ backend_name }} backend:

diff --git a/docs/backends/trino.md b/docs/backends/trino.md
@@ -3,6 +3,7 @@ backend_name: Trino
 backend_url: https://trino.io
 backend_module: trino
 exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
+memtable_impl: fallback
 ---
 
 # Trino
@@ -13,6 +14,8 @@ exports: ["PyArrow", "Parquet", "CSV", "Pandas"]
 
     The Trino backend is experimental and is subject to backwards incompatible changes.
 
+{% include 'backends/memtable-template.md' %}
+
 ## Install
 
 Install `ibis` and dependencies for the Trino backend:

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -54,6 +54,7 @@ plugins:
   - exclude:
       glob:
         - backends/template.md
+        - backends/memtable-template.md
         - backends/badges.md
         - backends/*_support_matrix.csv
         - backends/app/*