feat: add docs setup

.github/workflows/publish.yml

@@ -0,0 +1,98 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Publish Doc
+
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: '0 */2 * * *'  # every 2 hours
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: write
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  CARGO_BUILD_JOBS: "1"
+
+jobs:
+  build-and-deploy:
+    # scheduled run should only happen on main repo not forked ones.
+    if: |
+      github.event_name != 'schedule' || github.repository == 'lancedb/lance-python-doc'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-version: [ 3.11 ] # Ray does not support 3.12 yet.
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      # pin the toolchain version to avoid surprises
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+      - uses: rui314/setup-mold@v1
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Checkout lance-python-doc repo
+        uses: actions/checkout@v4
+      - name: Checkout lance repo
+        uses: actions/checkout@v4
+        with:
+          repository: lancedb/lance
+          path: lance
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: lance/python
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Copy docs to Lance
+        run: |
+          mv docs lance/python/docs
+      - name: Set up Python virtual environment
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install maturin
+          pip install -r lance/python/docs/requirements.txt
+      - name: Build Doc
+        working-directory: lance/python
+        run: |
+          source ../../venv/bin/activate
+          maturin develop
+          sphinx-build -b html docs docs/_build/html
+      - name: Add .nojekyll
+        run: |
+          touch lance/python/docs/_build/html/.nojekyll
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: lance/python/docs/_build/html

.gitignore

@@ -0,0 +1,97 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Tracing files
+trace-*.json
+
+**/*~
+**/__pycache__
+build/
+_build/
+dist/
+*.egg-info/
+.python-version
+
+.idea
+cmake-build-*
+.vscode
+.DS_Store
+
+python/lance/_*.cpp
+
+bin/
+
+
+*.parquet
+*.parq
+
+python/thirdparty/arrow/
+python/wheels
+python/benchmark_data
+
+logs
+*.ckpt
+
+docs/_build
+docs/api/python
+
+**/.ipynb_checkpoints/
+docs/notebooks
+
+notebooks/sift
+notebooks/image_data/data
+benchmarks/sift/sift
+benchmarks/sift/sift.lance
+benchmarks/sift/lance_ivf*.csv
+**/sift.tar.gz
+
+wheelhouse
+
+# pandas testing
+.hypothesis
+
+
+**/df.json
+
+# Rust
+target
+**/sccache.log
+
+# c++ lsp
+.ccls-cache/
+
+python/venv
+test_data/venv
+
+**/*.profraw
+*.lance

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/all-modules.rst

@@ -0,0 +1,14 @@
+All Modules
+===========
+
+.. automodule:: lance
+   :members:
+   :undoc-members:
+
+.. automodule:: lance.dataset
+   :members:
+   :undoc-members:
+
+.. automodule:: lance.fragment
+   :members:
+   :undoc-members:

docs/basic-io.rst

@@ -0,0 +1,13 @@
+Basic IOs
+=========
+
+The following functions are used to read and write data in Lance format.
+
+.. automethod:: lance.dataset.LanceDataset.insert
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.scanner
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.to_batches
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.to_table
+    :noindex:

docs/conf.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+# Configuration file for the Sphinx documentation builder.
+
+import sys
+import os
+
+# -- Project information -----------------------------------------------------
+
+project = "pylance"
+copyright = "%Y, Lance Developer"
+author = "Lance Developer"
+
+sys.path.insert(0, os.path.abspath("../"))
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.duration',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    'sphinx.ext.autosummary',
+    "sphinx.ext.napoleon",
+]
+
+napoleon_google_docstring = False
+napoleon_numpy_docstring = True
+napoleon_include_private_with_doc = False
+napoleon_include_special_with_doc = False
+
+autodoc_typehints = "signature"
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3/", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master/", None),
+    "numpy": ("https://numpy.org/doc/stable/", None),
+    "pyarrow": ("https://arrow.apache.org/docs/", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
+    "ray": ("https://docs.ray.io/en/latest/", None),
+}
+intersphinx_disabled_domains = ['std']
+
+# -- Options for HTML output -------------------------------------------------
+
+html_theme = 'sphinx_rtd_theme'
+html_theme_options = {
+    "collapse_navigation": False,   # Show all entries expanded
+    "navigation_depth": 4,          # Show nested headings
+    "titles_only": False            # Show both page titles and section titles
+}
+
+# -- doctest configuration ---------------------------------------------------
+
+doctest_global_setup = """
+import os
+import shutil
+from typing import Iterator
+
+import lance
+import pyarrow as pa
+import numpy as np
+import pandas as pd
+"""
+
+# Only test code examples in rst files
+doctest_test_doctest_blocks = ""

docs/data-evolution.rst

@@ -0,0 +1,10 @@
+Data Evolution
+==============
+
+Lance supports zero-copy data evolution, which means that you can add new columns and
+backfill column data to the dataset cheaply.
+
+.. automethod:: lance.dataset.LanceDataset.add_columns
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.drop_columns
+    :noindex:

docs/dataset.rst

@@ -0,0 +1,8 @@
+Lance Dataset
+=============
+
+The core of Lance is the ``LanceDataset`` class. User can open a dataset by using
+:py:meth:`lance.dataset`.
+
+.. autofunction:: lance.dataset
+    :noindex:

docs/index-and-search.rst

@@ -0,0 +1,11 @@
+Indexing and Searching
+======================
+
+.. automethod:: lance.dataset.LanceDataset.create_index
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.create_scalar_index
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.drop_index
+    :noindex:
+.. automethod:: lance.dataset.LanceDataset.scanner
+    :noindex:

docs/index.rst

@@ -0,0 +1,17 @@
+pylance
+===========
+
+Lance is a modern columnar data format for ML and LLMs.
+This is the auto-generated Python documentation for its Python SDK `pylance`.
+For more detailed quickstart and user guide, please visit the `official Lance documentation website <https://lancedb.github.io/lance>`_.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents
+
+   dataset
+   basic-io
+   random-access
+   data-evolution
+   index-and-search
+   all-modules