Skip to content

feat: upgrade lance-namespace to 0.3.1 and add missing apis #5457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jackye1995 merged 6 commits into from
Dec 11, 2025
Merged

feat: upgrade lance-namespace to 0.3.1 and add missing apis #5457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6644681
feat: upgrade lance-namespace to 0.3.0 and add missing apis
jackye1995 Dec 11, 2025
1fecf2e
update again
jackye1995 Dec 11, 2025
9045115
fix clippy
jackye1995 Dec 11, 2025
c9bee01
clippy
jackye1995 Dec 11, 2025
4b3cf69
msrv
jackye1995 Dec 11, 2025
6850099
update lock
jackye1995 Dec 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ lance-io = { version = "=1.1.0-beta.2", path = "./rust/lance-io", default-featur
lance-linalg = { version = "=1.1.0-beta.2", path = "./rust/lance-linalg" }
lance-namespace = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace" }
lance-namespace-impls = { version = "=1.1.0-beta.2", path = "./rust/lance-namespace-impls" }
lance-namespace-reqwest-client = "0.0.18"
lance-namespace-reqwest-client = "0.3.1"
lance-table = { version = "=1.1.0-beta.2", path = "./rust/lance-table" }
lance-test-macros = { version = "=1.1.0-beta.2", path = "./rust/lance-test-macros" }
lance-testing = { version = "=1.1.0-beta.2", path = "./rust/lance-testing" }
Expand Down
6 changes: 4 additions & 2 deletions java/lance-jni/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions java/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,12 @@
<dependency>
<groupId>org.lance</groupId>
<artifactId>lance-namespace-core</artifactId>
<version>0.2.1</version>
<version>0.3.1</version>
</dependency>
<dependency>
<groupId>org.lance</groupId>
<artifactId>lance-namespace-apache-client</artifactId>
<version>0.2.1</version>
<version>0.3.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
4 changes: 2 additions & 2 deletions python/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion python/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[project]
name = "pylance"
dynamic = ["version"]
dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.2.1"]
dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.3.1"]
description = "python wrapper for Lance columnar format"
authors = [{ name = "Lance Devs", email = "dev@lance.org" }]
license = { file = "LICENSE" }
Expand Down
1 change: 1 addition & 0 deletions rust/lance-io/src/object_store/storage_options.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider {
let request = DescribeTableRequest {
id: Some(self.table_id.clone()),
version: None,
with_table_uri: None,
};

let response = self
Expand Down
68 changes: 25 additions & 43 deletions rust/lance-namespace-impls/src/dir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -777,11 +777,20 @@ impl LanceNamespace for DirectoryNamespace {
let arrow_schema: arrow_schema::Schema = lance_schema.into();
let json_schema = arrow_schema_to_json(&arrow_schema)?;
Ok(DescribeTableResponse {
table: Some(table_name),
namespace: request.id.as_ref().map(|id| {
if id.len() > 1 {
id[..id.len() - 1].to_vec()
} else {
vec![]
}
}),
version: Some(version as i64),
location: Some(table_uri),
location: Some(table_uri.clone()),
table_uri: Some(table_uri),
schema: Some(Box::new(json_schema)),
properties: None,
storage_options: self.storage_options.clone(),
stats: None,
})
}
Err(err) => {
Expand All @@ -793,11 +802,20 @@ impl LanceNamespace for DirectoryNamespace {
.unwrap_or(false)
{
Ok(DescribeTableResponse {
table: Some(table_name),
namespace: request.id.as_ref().map(|id| {
if id.len() > 1 {
id[..id.len() - 1].to_vec()
} else {
vec![]
}
}),
version: None,
location: Some(table_uri),
location: Some(table_uri.clone()),
table_uri: Some(table_uri),
schema: None,
properties: None,
storage_options: self.storage_options.clone(),
stats: None,
})
} else {
Err(Error::Namespace {
Expand Down Expand Up @@ -886,21 +904,6 @@ impl LanceNamespace for DirectoryNamespace {
});
}

// Validate location if provided
if let Some(location) = &request.location {
let location = location.trim_end_matches('/');
if location != table_uri {
return Err(Error::Namespace {
source: format!(
"Cannot create table {} at location {}, must be at location {}",
table_name, location, table_uri
)
.into(),
location: snafu::location!(),
});
}
}

// Parse the Arrow IPC stream from request_data
let cursor = Cursor::new(request_data.to_vec());
let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| Error::Namespace {
Expand Down Expand Up @@ -948,9 +951,9 @@ impl LanceNamespace for DirectoryNamespace {
})?;

Ok(CreateTableResponse {
transaction_id: None,
version: Some(1),
location: Some(table_uri),
properties: None,
storage_options: self.storage_options.clone(),
})
}
Expand Down Expand Up @@ -1007,6 +1010,7 @@ impl LanceNamespace for DirectoryNamespace {
})?;

Ok(CreateEmptyTableResponse {
transaction_id: None,
location: Some(table_uri),
properties: None,
storage_options: self.storage_options.clone(),
Expand Down Expand Up @@ -1188,28 +1192,6 @@ mod tests {
);
}

#[tokio::test]
async fn test_create_table_with_wrong_location() {
let (namespace, _temp_dir) = create_test_namespace().await;

// Create test IPC data
let schema = create_test_schema();
let ipc_data = create_test_ipc_data(&schema);

let mut request = CreateTableRequest::new();
request.id = Some(vec!["test_table".to_string()]);
request.location = Some("/wrong/path/table.lance".to_string());

let result = namespace
.create_table(request, bytes::Bytes::from(ipc_data))
.await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("must be at location"));
}

#[tokio::test]
async fn test_list_tables() {
let (namespace, _temp_dir) = create_test_namespace().await;
Expand Down Expand Up @@ -2360,7 +2342,7 @@ mod tests {
register_req.id = Some(vec!["registered_table".to_string()]);

let response = namespace.register_table(register_req).await.unwrap();
assert_eq!(response.location, "external_table.lance");
assert_eq!(response.location, Some("external_table.lance".to_string()));

// Verify table exists in namespace
let mut exists_req = TableExistsRequest::new();
Expand Down
45 changes: 24 additions & 21 deletions rust/lance-namespace-impls/src/dir/manifest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1069,6 +1069,14 @@ impl LanceNamespace for ManifestNamespace {
let object_id = Self::str_object_id(table_id);
let table_info = self.query_manifest_for_table(&object_id).await?;

// Extract table name and namespace from table_id
let table_name = table_id.last().cloned().unwrap_or_default();
let namespace_id: Vec<String> = if table_id.len() > 1 {
table_id[..table_id.len() - 1].to_vec()
} else {
vec![]
};

match table_info {
Some(info) => {
// Construct full URI from relative location
Expand All @@ -1088,21 +1096,27 @@ impl LanceNamespace for ManifestNamespace {
let json_schema = arrow_schema_to_json(&arrow_schema)?;

Ok(DescribeTableResponse {
table: Some(table_name.clone()),
namespace: Some(namespace_id.clone()),
version: Some(version as i64),
location: Some(table_uri),
location: Some(table_uri.clone()),
table_uri: Some(table_uri),
schema: Some(Box::new(json_schema)),
properties: None,
storage_options: self.storage_options.clone(),
stats: None,
})
}
Err(_) => {
// If dataset can't be opened (e.g., empty table), return minimal info
Ok(DescribeTableResponse {
table: Some(table_name),
namespace: Some(namespace_id),
version: None,
location: Some(table_uri),
location: Some(table_uri.clone()),
table_uri: Some(table_uri),
schema: None,
properties: None,
storage_options: self.storage_options.clone(),
stats: None,
})
}
}
Expand Down Expand Up @@ -1188,21 +1202,6 @@ impl LanceNamespace for ManifestNamespace {
});
}

// Validate location if provided
if let Some(location) = &request.location {
let location = location.trim_end_matches('/');
if location != table_uri {
return Err(Error::Namespace {
source: format!(
"Cannot create table {} at location {}, must be at location {}",
table_name, location, table_uri
)
.into(),
location: location!(),
});
}
}

// Write the data using Lance Dataset
let cursor = Cursor::new(data.to_vec());
let stream_reader = StreamReader::try_new(cursor, None)
Expand Down Expand Up @@ -1241,9 +1240,9 @@ impl LanceNamespace for ManifestNamespace {
.await?;

Ok(CreateTableResponse {
transaction_id: None,
version: Some(1),
location: Some(table_uri),
properties: None,
storage_options: self.storage_options.clone(),
})
}
Expand Down Expand Up @@ -1431,6 +1430,7 @@ impl LanceNamespace for ManifestNamespace {
.await?;

Ok(CreateNamespaceResponse {
transaction_id: None,
properties: request.properties,
})
}
Expand Down Expand Up @@ -1613,6 +1613,7 @@ impl LanceNamespace for ManifestNamespace {
);

Ok(CreateEmptyTableResponse {
transaction_id: None,
location: Some(table_uri),
properties: None,
storage_options: self.storage_options.clone(),
Expand Down Expand Up @@ -1688,7 +1689,8 @@ impl LanceNamespace for ManifestNamespace {
.await?;

Ok(RegisterTableResponse {
location,
transaction_id: None,
location: Some(location),
properties: None,
})
}
Expand Down Expand Up @@ -1732,6 +1734,7 @@ impl LanceNamespace for ManifestNamespace {
};

Ok(DeregisterTableResponse {
transaction_id: None,
id: request.id.clone(),
location: Some(table_uri),
properties: None,
Expand Down
Loading