Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ public CreateTableResponse createTable(CreateTableRequest request, byte[] reques
// Create the Lance dataset with data
Dataset.create(allocator, tablePath, schema, writeParams);
CreateTableResponse response = new CreateTableResponse();
response.setLocation(tablePath);
response.setLocation(OpenDalUtil.denormalizeUri(tablePath));
response.setVersion(1L);
response.setStorageOptions(config.getStorageOptions());
return response;
Expand Down Expand Up @@ -188,7 +188,7 @@ public CreateEmptyTableResponse createEmptyTable(CreateEmptyTableRequest request
}

CreateEmptyTableResponse response = new CreateEmptyTableResponse();
response.setLocation(tablePath);
response.setLocation(OpenDalUtil.denormalizeUri(tablePath));
response.setStorageOptions(config.getStorageOptions());
return response;
}
Expand All @@ -203,7 +203,7 @@ public DropTableResponse dropTable(DropTableRequest request) {
try {
Dataset.drop(tablePath, config.getStorageOptions());
DropTableResponse response = new DropTableResponse();
response.setLocation(tablePath);
response.setLocation(OpenDalUtil.denormalizeUri(tablePath));
response.setId(request.getId());
return response;
} catch (Exception e) {
Expand Down Expand Up @@ -337,7 +337,7 @@ public DescribeTableResponse describeTable(DescribeTableRequest request) {
}

DescribeTableResponse response = new DescribeTableResponse();
response.setLocation(tablePath);
response.setLocation(OpenDalUtil.denormalizeUri(tablePath));
response.setStorageOptions(config.getStorageOptions());
return response;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ public static String normalizeScheme(String scheme) {
case "s3n":
return "s3";
case "abfs":
case "az":
return "azblob";
case "file":
return "fs";
Expand All @@ -95,6 +96,51 @@ public static String normalizeScheme(String scheme) {
}
}

/**
* Denormalize scheme names to Lance-compatible format. This is the reverse of normalizeScheme,
* converting OpenDAL schemes back to Lance schemes.
*
* @param scheme the OpenDAL scheme
* @return denormalized scheme for Lance
*/
public static String denormalizeScheme(String scheme) {
switch (scheme.toLowerCase()) {
case "azblob":
return "az";
default:
return scheme.toLowerCase();
}
}

/**
* Convert an OpenDAL-compatible URI to a Lance-compatible URI. This converts schemes like
* azblob:// back to az:// for Lance operations.
*
* @param uri the OpenDAL-compatible URI
* @return Lance-compatible URI
*/
public static String denormalizeUri(String uri) {
if (uri == null) {
return null;
}

String[] schemeSplit = uri.split("://", 2);
if (schemeSplit.length < 2) {
// No scheme, return as-is
return uri;
}

String scheme = schemeSplit[0];
String denormalizedScheme = denormalizeScheme(scheme);

if (denormalizedScheme.equals(scheme)) {
// No change needed
return uri;
}

return denormalizedScheme + "://" + schemeSplit[1];
}

public static String stripTrailingSlash(String path) {
if (path == null) {
return null;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.lancedb.lance.namespace.util;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

public class TestOpenDalUtil {

@Test
public void testNormalizeScheme() {
// Test s3 aliases
assertEquals("s3", OpenDalUtil.normalizeScheme("s3"));
assertEquals("s3", OpenDalUtil.normalizeScheme("s3a"));
assertEquals("s3", OpenDalUtil.normalizeScheme("s3n"));
assertEquals("s3", OpenDalUtil.normalizeScheme("S3A"));

// Test Azure aliases
assertEquals("azblob", OpenDalUtil.normalizeScheme("azblob"));
assertEquals("azblob", OpenDalUtil.normalizeScheme("az"));
assertEquals("azblob", OpenDalUtil.normalizeScheme("abfs"));
assertEquals("azblob", OpenDalUtil.normalizeScheme("AZ"));
assertEquals("azblob", OpenDalUtil.normalizeScheme("ABFS"));

// Test filesystem
assertEquals("fs", OpenDalUtil.normalizeScheme("file"));
assertEquals("fs", OpenDalUtil.normalizeScheme("fs"));
assertEquals("fs", OpenDalUtil.normalizeScheme("FILE"));

// Test GCS (no aliases)
assertEquals("gcs", OpenDalUtil.normalizeScheme("gcs"));
assertEquals("gcs", OpenDalUtil.normalizeScheme("GCS"));

// Test unknown scheme
assertEquals("hdfs", OpenDalUtil.normalizeScheme("hdfs"));
assertEquals("custom", OpenDalUtil.normalizeScheme("custom"));
}

@Test
public void testDenormalizeScheme() {
// Test Azure denormalization
assertEquals("az", OpenDalUtil.denormalizeScheme("azblob"));
assertEquals("az", OpenDalUtil.denormalizeScheme("AZBLOB"));

// Test schemes that don't need denormalization
assertEquals("s3", OpenDalUtil.denormalizeScheme("s3"));
assertEquals("gcs", OpenDalUtil.denormalizeScheme("gcs"));
assertEquals("fs", OpenDalUtil.denormalizeScheme("fs"));
assertEquals("file", OpenDalUtil.denormalizeScheme("file"));
assertEquals("hdfs", OpenDalUtil.denormalizeScheme("hdfs"));
}

@Test
public void testDenormalizeUri() {
// Test Azure URI denormalization
assertEquals(
"az://container/path/to/table.lance",
OpenDalUtil.denormalizeUri("azblob://container/path/to/table.lance"));
assertEquals(
"az://mycontainer/data/mytable.lance",
OpenDalUtil.denormalizeUri("azblob://mycontainer/data/mytable.lance"));

// Test URIs that don't need denormalization
assertEquals(
"s3://bucket/path/to/table.lance",
OpenDalUtil.denormalizeUri("s3://bucket/path/to/table.lance"));
assertEquals(
"gcs://bucket/path/to/table.lance",
OpenDalUtil.denormalizeUri("gcs://bucket/path/to/table.lance"));
assertEquals(
"file:///local/path/to/table.lance",
OpenDalUtil.denormalizeUri("file:///local/path/to/table.lance"));
assertEquals(
"/local/path/to/table.lance", OpenDalUtil.denormalizeUri("/local/path/to/table.lance"));

// Test null and edge cases
assertNull(OpenDalUtil.denormalizeUri(null));
assertEquals("", OpenDalUtil.denormalizeUri(""));
assertEquals("noscheme", OpenDalUtil.denormalizeUri("noscheme"));
}

@Test
public void testStripTrailingSlash() {
// Test with trailing slashes
assertEquals("/path/to/dir", OpenDalUtil.stripTrailingSlash("/path/to/dir/"));
assertEquals("/path/to/dir", OpenDalUtil.stripTrailingSlash("/path/to/dir//"));
assertEquals("s3://bucket/path", OpenDalUtil.stripTrailingSlash("s3://bucket/path///"));

// Test without trailing slashes
assertEquals("/path/to/dir", OpenDalUtil.stripTrailingSlash("/path/to/dir"));
assertEquals("s3://bucket/path", OpenDalUtil.stripTrailingSlash("s3://bucket/path"));

// Test null and edge cases
assertNull(OpenDalUtil.stripTrailingSlash(null));
assertEquals("", OpenDalUtil.stripTrailingSlash("/"));
assertEquals("", OpenDalUtil.stripTrailingSlash("///"));
}
}