Python Replace Datasets

A table has no update in place. To change datasets, delete the old datasets and add the new datasets.

Example files and table for this page

The examples use small NetCDF files of CTD profiles, see Getting started. Run this block first. It writes the files and makes the table profiles.lance from the six files in profiles/.

from pathlib import Path

import numpy as np
import pandas as pd
import xarray as xr


def write_profile(path, station, cruise, lat, lon, levels, time, temp_type="float64", extra=None):
    """Write one CTD profile as a NetCDF file."""
    depth = np.arange(levels, dtype="float64") * 10.0
    ds = xr.Dataset(
        {
            "TEMP": ("depth", (15.0 - depth / 100.0).astype(temp_type), {"units": "degree_Celsius"}),
            "PSAL": ("depth", 35.0 + depth / 1000.0, {"units": "psu"}),
            "LATITUDE": ((), lat),
            "LONGITUDE": ((), lon),
            "TIME": ((), pd.Timestamp(time)),
        },
        coords={"depth": ("depth", depth, {"units": "m"})},
        attrs={"station": station, "cruise": cruise},
    )
    if extra:
        ds["DOXY"] = ("depth", np.full(levels, 250.0), {"units": "umol/kg"})
    Path(path).parent.mkdir(exist_ok=True)
    ds.to_netcdf(path)


# Six profiles of two cruises.
write_profile("profiles/A1.nc", "A1", "cruise-1", 52.5, 4.0, 50, "2026-06-01")
write_profile("profiles/A2.nc", "A2", "cruise-1", 53.0, 4.5, 80, "2026-06-02")
write_profile("profiles/A3.nc", "A3", "cruise-1", 60.0, 2.0, 120, "2026-06-03")
write_profile("profiles/B1.nc", "B1", "cruise-2", 54.0, 3.0, 60, "2026-07-01")
write_profile("profiles/B2.nc", "B2", "cruise-2", 55.5, 3.5, 70, "2026-07-02")
write_profile("profiles/B3.nc", "B3", "cruise-2", 57.0, 5.0, 90, "2026-07-03")
# Three new profiles of a third cruise.
write_profile("new/C1.nc", "C1", "cruise-3", 58.0, 6.0, 40, "2026-08-01")
write_profile("new/C2.nc", "C2", "cruise-3", 58.5, 6.5, 45, "2026-08-02")
write_profile("new/C3.nc", "C3", "cruise-3", 59.0, 7.0, 50, "2026-08-03")
# Corrected profiles, with other depth levels.
write_profile("corrected/A1.nc", "A1", "cruise-1", 52.5, 4.0, 55, "2026-06-01")
write_profile("corrected/A2.nc", "A2", "cruise-1", 53.0, 4.5, 90, "2026-06-02")
write_profile("corrected/B1.nc", "B1", "cruise-2", 54.0, 3.0, 65, "2026-07-01")
write_profile("corrected/B2.nc", "B2", "cruise-2", 55.5, 3.5, 75, "2026-07-02")
# Files that differ from the others.
write_profile("other/int_temp.nc", "D1", "cruise-4", 50.0, 1.0, 10, "2026-09-01", temp_type="int32")
write_profile("other/with_doxy.nc", "E1", "cruise-5", 51.0, 2.0, 20, "2026-10-01", extra=True)
write_profile("other/float32_temp.nc", "F1", "cruise-6", 52.0, 2.0, 10, "2026-11-01", temp_type="float32")

import lance_nd


def stations(table):
    """The station of each dataset in the table, in table order."""
    return [batch.to_xarray().attrs["station"] for batch in table.scan(project=[])]


with lance_nd.create("profiles.lance", mode="overwrite", stats=True) as writer:
    for path in sorted(Path("profiles").glob("*.nc")):
        with xr.open_dataset(path) as ds:
            writer.write(ds)
table = lance_nd.open("profiles.lance")

Replace one dataset

Here station A2 gets a corrected file, corrected/A2.nc, with 90 depth levels.

table.delete("__global_attrs__.station = 'A2'")
with lance_nd.append("profiles.lance") as writer:
    with xr.open_dataset("corrected/A2.nc") as ds:
        writer.write(ds)

table = lance_nd.open("profiles.lance")
print(table.version, table.count_rows())
print(stations(table))

3 6
['A1', 'A3', 'B1', 'B2', 'B3', 'A2']

• The delete and the append are two commits, so the table gets two versions.
• The new A2 comes at the end of the table. Datasets have no fixed position. Use an attribute, such as station, to find a dataset.

Replace many datasets

Delete all old datasets with one filter, then add all new datasets with one writer. This is two commits for any number of datasets. Here the stations come from the names of the corrected files.

files = [Path("corrected/B1.nc"), Path("corrected/B2.nc")]
names = ", ".join(f"'{path.stem}'" for path in files)

table.delete(f"__global_attrs__.station IN ({names})")
with lance_nd.append("profiles.lance") as writer:
    for path in files:
        with xr.open_dataset(path) as ds:
            writer.write(ds)

table = lance_nd.open("profiles.lance")
print(table.version, table.count_rows())
print([(b.to_xarray().attrs["station"], b.to_xarray().sizes["depth"]) for b in table.scan(project=["depth"])])

5 6
[('A1', 50), ('A3', 120), ('B3', 90), ('A2', 90), ('B1', 65), ('B2', 75)]

Between the two commits

• A reader that opens the table after the delete and before the append sees the table without the old datasets and without the new datasets.
• When the append fails, the old datasets stay deleted. Write the new datasets again, or undo the delete with restore().
• Do the delete first. When you append first, the filter of the delete also matches the new datasets.

A safe pattern keeps the version before the delete:

before = table.version
table.delete("__global_attrs__.station = 'A1'")
try:
    with lance_nd.append("profiles.lance") as writer:
        with xr.open_dataset("corrected/A1.nc") as ds:
            writer.write(ds)
except lance_nd.LanceNdError:
    table.restore(before)
    raise

print(lance_nd.open("profiles.lance").count_rows())

6

Many small replacements

Each replacement makes two versions and a small fragment. After many replacements, compact the table to merge the small fragments and to remove the deleted rows.