Python Create a Table

lance_nd.create() makes a new table and gives a writer for its first datasets. The table exists only after the first commit.

Example files and table for this page

The examples use small NetCDF files of CTD profiles, see Getting started. Run this block first. It writes the files and makes the table profiles.lance from the six files in profiles/.

from pathlib import Path

import numpy as np
import pandas as pd
import xarray as xr


def write_profile(path, station, cruise, lat, lon, levels, time, temp_type="float64", extra=None):
    """Write one CTD profile as a NetCDF file."""
    depth = np.arange(levels, dtype="float64") * 10.0
    ds = xr.Dataset(
        {
            "TEMP": ("depth", (15.0 - depth / 100.0).astype(temp_type), {"units": "degree_Celsius"}),
            "PSAL": ("depth", 35.0 + depth / 1000.0, {"units": "psu"}),
            "LATITUDE": ((), lat),
            "LONGITUDE": ((), lon),
            "TIME": ((), pd.Timestamp(time)),
        },
        coords={"depth": ("depth", depth, {"units": "m"})},
        attrs={"station": station, "cruise": cruise},
    )
    if extra:
        ds["DOXY"] = ("depth", np.full(levels, 250.0), {"units": "umol/kg"})
    Path(path).parent.mkdir(exist_ok=True)
    ds.to_netcdf(path)


# Six profiles of two cruises.
write_profile("profiles/A1.nc", "A1", "cruise-1", 52.5, 4.0, 50, "2026-06-01")
write_profile("profiles/A2.nc", "A2", "cruise-1", 53.0, 4.5, 80, "2026-06-02")
write_profile("profiles/A3.nc", "A3", "cruise-1", 60.0, 2.0, 120, "2026-06-03")
write_profile("profiles/B1.nc", "B1", "cruise-2", 54.0, 3.0, 60, "2026-07-01")
write_profile("profiles/B2.nc", "B2", "cruise-2", 55.5, 3.5, 70, "2026-07-02")
write_profile("profiles/B3.nc", "B3", "cruise-2", 57.0, 5.0, 90, "2026-07-03")
# Three new profiles of a third cruise.
write_profile("new/C1.nc", "C1", "cruise-3", 58.0, 6.0, 40, "2026-08-01")
write_profile("new/C2.nc", "C2", "cruise-3", 58.5, 6.5, 45, "2026-08-02")
write_profile("new/C3.nc", "C3", "cruise-3", 59.0, 7.0, 50, "2026-08-03")
# Corrected profiles, with other depth levels.
write_profile("corrected/A1.nc", "A1", "cruise-1", 52.5, 4.0, 55, "2026-06-01")
write_profile("corrected/A2.nc", "A2", "cruise-1", 53.0, 4.5, 90, "2026-06-02")
write_profile("corrected/B1.nc", "B1", "cruise-2", 54.0, 3.0, 65, "2026-07-01")
write_profile("corrected/B2.nc", "B2", "cruise-2", 55.5, 3.5, 75, "2026-07-02")
# Files that differ from the others.
write_profile("other/int_temp.nc", "D1", "cruise-4", 50.0, 1.0, 10, "2026-09-01", temp_type="int32")
write_profile("other/with_doxy.nc", "E1", "cruise-5", 51.0, 2.0, 20, "2026-10-01", extra=True)
write_profile("other/float32_temp.nc", "F1", "cruise-6", 52.0, 2.0, 10, "2026-11-01", temp_type="float32")

import lance_nd


def stations(table):
    """The station of each dataset in the table, in table order."""
    return [batch.to_xarray().attrs["station"] for batch in table.scan(project=[])]


with lance_nd.create("profiles.lance", mode="overwrite", stats=True) as writer:
    for path in sorted(Path("profiles").glob("*.nc")):
        with xr.open_dataset(path) as ds:
            writer.write(ds)
table = lance_nd.open("profiles.lance")

Create a table

Open each NetCDF file and write its dataset. Each file becomes one row.

files = sorted(Path("profiles").glob("*.nc"))

with lance_nd.create("new.lance") as writer:
    for path in files:
        with xr.open_dataset(path) as ds:
            writer.write(ds)

table = writer.table
print(table.version, table.count_rows())

1 6

• The writer commits all datasets at the end of the with block, in one version.
• writer.table is the table after the commit.
• When the block raises an error, the writer commits nothing and the table does not exist.
• Write each dataset inside its with xr.open_dataset(...) block. The writer reads the values from the file, and the file then closes.

Without a with block

Call finish() to commit, or abort() to drop the datasets.

writer = lance_nd.create("manual.lance")
for path in files[:2]:
    with xr.open_dataset(path) as ds:
        writer.write(ds)
table = writer.finish()
print(table.count_rows())

2

A writer that you do not finish commits nothing.

Store stats

stats=True stores the minimum and the maximum of each variable for each dataset. A scan with prune uses them to skip datasets, see Read datasets.

with lance_nd.create("stats.lance", stats=True) as writer:
    for path in files:
        with xr.open_dataset(path) as ds:
            writer.write(ds)
print(writer.table.has_stats)

True

• Set stats when you create the table. Each append then stores stats too.
• Stats need a little disk space: one small column for each variable.

Compression

column_defaults sets the encoding of all columns. column_options sets the encoding of one variable and overrides the defaults.

with lance_nd.create(
    "zstd.lance",
    column_defaults={"compression": "zstd", "compression_level": 3},
    column_options={"TEMP": {"byte_stream_split": "on"}},
) as writer:
    for path in files:
        with xr.open_dataset(path) as ds:
            writer.write(ds)
print(writer.table.count_rows())

6

Key	Values
compression	"none", "lz4", "zstd"
compression_level	an integer, with "zstd" only
byte_stream_split	"off", "on", "auto"
rle_threshold	a float
structural_encoding	"miniblock", "fullzip"

"zstd" gives small files for most scientific data. byte_stream_split often makes float values smaller.

Replace a table

create() raises TableExistsError when the table exists. mode="overwrite" replaces the table at the commit.

try:
    lance_nd.create("new.lance")
except lance_nd.TableExistsError as error:
    print(type(error).__name__)

with lance_nd.create("new.lance", mode="overwrite") as writer:
    with xr.open_dataset("profiles/A1.nc") as ds:
        writer.write(ds)
print(writer.table.count_rows(), writer.table.version)

TableExistsError
1 2

• The overwrite is a new version, so you can still read the old content, see Versions.
• Readers see the old table until the commit.

Fragment size

A writer collects datasets in memory, then writes them as one fragment of about 128 MiB on disk. Up to 4 fragments are written at the same time.

lance_nd.create("big.lance", target_fragment_bytes=512 << 20, write_parallelism=8)

• A larger target_fragment_bytes gives fewer, larger files. A scan opens fewer files, but the writer uses more memory.
• The writer estimates the size on disk, so the sizes are not exact. Compact gives exact sizes later.

Other options

• casting decides what happens when a variable gets another type, see Add datasets.
• storage_options writes to S3, Google Cloud Storage or Azure, see Object stores.
• The API reference lists all arguments.