""" Minimal Complete Example: Icechunk with file:/// URLs This script demonstrates: 1. The error when opening an icechunk repository with file:/// virtual chunk URLs 2. How to fix it by adding file:/// credentials to the containers_credentials """ import icechunk import xarray import tempfile import os import numpy as np import zarr def create_test_repo_with_file_urls(repo_path, data_dir): """ Create a test icechunk repository with virtual chunks pointing to file:/// URLs. Args: repo_path: Path where the icechunk repository will be created data_dir: Directory containing the actual data files (will be referenced via file:///) """ # Create actual chunk data files that will be referenced via virtual refs chunk_file = os.path.join(data_dir, "chunk_0_0.bin") chunk_data = np.random.rand(5, 5).astype("f4") chunk_data.tofile(chunk_file) # Create icechunk repository storage = icechunk.local_filesystem_storage(repo_path) # Configure with virtual chunk container pointing to file:/// URL config = icechunk.RepositoryConfig.default() file_url_prefix = f"file://{data_dir}/" # For file:// URLs, pass the base path to local_filesystem_store config.set_virtual_chunk_container( icechunk.VirtualChunkContainer( file_url_prefix, icechunk.local_filesystem_store(data_dir), ) ) # IMPORTANT: Add credentials for file:/// URLs # For file:// URLs, we can pass None as credentials (local files don't need auth) credentials = icechunk.containers_credentials({ file_url_prefix: None }) # Create repository repo = icechunk.Repository.create(storage, config, credentials) session = repo.writable_session("main") # Create a simple array with proper metadata for xarray group = zarr.open_group(session.store, mode="a") arr = group.create_array( "test_arr", shape=(10, 10), dtype="f4", chunks=(5, 5), dimension_names=["dim0", "dim1"], # Required for xarray ) # Set virtual refs pointing to the file:/// URLs # This is what triggers the authorization requirement chunk_size = 5 * 5 * 4 # 5x5 array of float32 = 100 bytes virtual_refs = [ icechunk.VirtualChunkSpec( index=[0, 0], # First chunk location=f"{file_url_prefix}chunk_0_0.bin", length=chunk_size, offset=0, ), # Add more chunks if needed - for simplicity, just one chunk ] session.store.set_virtual_refs("test_arr", virtual_refs) repo.save_config() session.commit("initial commit") return repo_path, file_url_prefix def open_repo_without_file_credentials(repo_path, file_url_prefix): """ This will FAIL with authorization error because file:/// credentials are missing and virtual refs point to file:/// URLs. """ storage = icechunk.local_filesystem_storage(repo_path) # Need to configure the virtual chunk container # Extract the path from file:// URL (remove file:// prefix and trailing /) data_path = file_url_prefix.replace("file://", "").rstrip("/") config = icechunk.RepositoryConfig.default() config.set_virtual_chunk_container( icechunk.VirtualChunkContainer( file_url_prefix, icechunk.local_filesystem_store(data_path), ) ) # Missing file:/// credentials - this causes the error! credentials = None try: repo = icechunk.Repository.open(storage, config, credentials) session = repo.readonly_session("main") ds = xarray.open_zarr(session.store, chunks=None) # Try to actually load the data - this should trigger the authorization error print("Attempting to load data...") data = ds.test_arr[0:5, 0:5].load() # Load the chunk print(f"Success (unexpected!): Loaded data shape {data.shape}") return ds except Exception as e: error_msg = str(e) if "authorize" in error_msg.lower() or "credential" in error_msg.lower(): print(f"✓ AUTHORIZATION ERROR (expected): {error_msg}") else: print(f"Other error: {error_msg}") return None def open_repo_with_file_credentials(repo_path, file_url_prefix): """ This will SUCCEED because file:/// credentials are provided. """ storage = icechunk.local_filesystem_storage(repo_path) # Configure the virtual chunk container # Extract the path from file:// URL (remove file:// prefix and trailing /) data_path = file_url_prefix.replace("file://", "").rstrip("/") config = icechunk.RepositoryConfig.default() config.set_virtual_chunk_container( icechunk.VirtualChunkContainer( file_url_prefix, icechunk.local_filesystem_store(data_path), ) ) # Add credentials for file:/// URLs - this fixes the issue! # For file:// URLs, we can pass None as credentials (local files don't need auth) credentials = icechunk.containers_credentials({ file_url_prefix: None }) repo = icechunk.Repository.open(storage, config, credentials) session = repo.readonly_session("main") ds = xarray.open_zarr(session.store, chunks=None) # Try to actually load the data - this should work now (authorization-wise) # Note: The actual data loading may fail due to format issues, but authorization should pass print("Attempting to load data...") try: data = ds.test_arr[0:5, 0:5].load() # Load the chunk print(f"✓ Success! Loaded data shape: {data.shape}") except Exception as e: # If it's not an authorization error, that's fine - we've proven authorization works if "authorize" not in str(e).lower() and "credential" not in str(e).lower(): print(f"✓ Authorization passed! (Data format error is expected: {type(e).__name__})") else: raise return ds if __name__ == "__main__": with tempfile.TemporaryDirectory() as tmpdir: repo_path = os.path.join(tmpdir, "test_repo") data_dir = os.path.join(tmpdir, "data") os.makedirs(data_dir) print("Creating test repository with file:/// URLs...") repo_path, file_url_prefix = create_test_repo_with_file_urls(repo_path, data_dir) print("\n" + "="*60) print("Attempting to open WITHOUT file:/// credentials:") print("="*60) open_repo_without_file_credentials(repo_path, file_url_prefix) print("\n" + "="*60) print("Attempting to open WITH file:/// credentials:") print("="*60) ds = open_repo_with_file_credentials(repo_path, file_url_prefix) if ds is not None: print(f"Dataset shape: {ds.test_arr.shape}")