diff --git a/src/lsdb/loaders/hats/read_hats.py b/src/lsdb/loaders/hats/read_hats.py index 2f1534dd3..9f5ccbed2 100644 --- a/src/lsdb/loaders/hats/read_hats.py +++ b/src/lsdb/loaders/hats/read_hats.py @@ -309,6 +309,8 @@ def _load_object_catalog(hc_catalog, config): def _generate_pyarrow_filters_from_moc(filtered_catalog): pyarrow_filter = [] + if SPATIAL_INDEX_COLUMN not in filtered_catalog.schema.names: + return pyarrow_filter if filtered_catalog.moc is not None: depth_array = filtered_catalog.moc.to_depth29_ranges if len(depth_array) > MAX_PYARROW_FILTERS: diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 3b2047f80..890f2bdee 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -402,6 +402,7 @@ " catalog_type=\"source\",\n", " output_artifact_name=\"small_sky_source\",\n", " highest_healpix_order=2,\n", + " add_healpix_29=False,\n", " pixel_threshold=3000,\n", " tmp_dir=pipeline_tmp,\n", " )\n", @@ -873,7 +874,7 @@ ], "metadata": { "kernelspec": { - "display_name": "lsdb", + "display_name": "sep", "language": "python", "name": "python3" }, @@ -887,7 +888,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.11" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/tests/data/small_sky_source/dataset/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_source/dataset/Norder=1/Dir=0/Npix=47.parquet index 2e31f2d1d..0ee7a7afa 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=1/Dir=0/Npix=47.parquet and b/tests/data/small_sky_source/dataset/Norder=1/Dir=0/Npix=47.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=176.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=176.parquet index 470ca7085..9d73645a0 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=176.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=176.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=177.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=177.parquet index ca98319b2..245a7cec9 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=177.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=177.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=178.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=178.parquet index 3d45918db..63621dcaa 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=178.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=178.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=179.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=179.parquet index a7cac98d8..006c34b6e 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=179.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=179.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=180.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=180.parquet index 7add6a485..733ab9db4 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=180.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=180.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=181.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=181.parquet index dd596866f..71b9b489a 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=181.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=181.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=182.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=182.parquet index c2d1a8653..c230396a0 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=182.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=182.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=183.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=183.parquet index fe5427416..06db09291 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=183.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=183.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=184.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=184.parquet index 78847b525..fb8e8a598 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=184.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=184.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=185.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=185.parquet index 93b5c2606..a5617e570 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=185.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=185.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=186.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=186.parquet index 8297c4d4e..133399ebe 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=186.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=186.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=187.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=187.parquet index 8f6231556..30d5cb9eb 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=187.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=187.parquet differ diff --git a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=66.parquet b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=66.parquet index 6c8f6ff79..56f15cd5a 100644 Binary files a/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=66.parquet and b/tests/data/small_sky_source/dataset/Norder=2/Dir=0/Npix=66.parquet differ diff --git a/tests/data/small_sky_source/dataset/_common_metadata b/tests/data/small_sky_source/dataset/_common_metadata index b046e06b4..750a26844 100644 Binary files a/tests/data/small_sky_source/dataset/_common_metadata and b/tests/data/small_sky_source/dataset/_common_metadata differ diff --git a/tests/data/small_sky_source/dataset/_metadata b/tests/data/small_sky_source/dataset/_metadata index eace9df94..f3c3a0d1f 100644 Binary files a/tests/data/small_sky_source/dataset/_metadata and b/tests/data/small_sky_source/dataset/_metadata differ diff --git a/tests/data/small_sky_source/dataset/data_thumbnail.parquet b/tests/data/small_sky_source/dataset/data_thumbnail.parquet deleted file mode 100644 index 83f73e6eb..000000000 Binary files a/tests/data/small_sky_source/dataset/data_thumbnail.parquet and /dev/null differ diff --git a/tests/data/small_sky_source/hats.properties b/tests/data/small_sky_source/hats.properties index 02dc0a527..4f33ef6c1 100644 --- a/tests/data/small_sky_source/hats.properties +++ b/tests/data/small_sky_source/hats.properties @@ -6,9 +6,9 @@ hats_col_ra=source_ra hats_col_dec=source_dec hats_npix_suffix=.parquet hats_skymap_order=2 -hats_builder=hats-import v0.6.1.dev1+gd082cb2, hats v0.7.dev0+g8d490830.d20250613 -hats_creation_date=2025-06-16T15\:26UTC -hats_estsize=819 +hats_builder=hats-import v0.6.5.dev5+g1a95e2449, hats v0.6.5.dev2+ga49f49815 +hats_creation_date=2025-09-08T13:40UTC +hats_estsize=692 hats_release_date=2024-09-18 hats_version=v0.1 hats_max_rows=3000 diff --git a/tests/data/small_sky_source/properties b/tests/data/small_sky_source/properties index 02dc0a527..4f33ef6c1 100644 --- a/tests/data/small_sky_source/properties +++ b/tests/data/small_sky_source/properties @@ -6,9 +6,9 @@ hats_col_ra=source_ra hats_col_dec=source_dec hats_npix_suffix=.parquet hats_skymap_order=2 -hats_builder=hats-import v0.6.1.dev1+gd082cb2, hats v0.7.dev0+g8d490830.d20250613 -hats_creation_date=2025-06-16T15\:26UTC -hats_estsize=819 +hats_builder=hats-import v0.6.5.dev5+g1a95e2449, hats v0.6.5.dev2+ga49f49815 +hats_creation_date=2025-09-08T13:40UTC +hats_estsize=692 hats_release_date=2024-09-18 hats_version=v0.1 hats_max_rows=3000 diff --git a/tests/lsdb/loaders/hats/test_read_hats.py b/tests/lsdb/loaders/hats/test_read_hats.py index 1c016895f..92d51a9d2 100644 --- a/tests/lsdb/loaders/hats/test_read_hats.py +++ b/tests/lsdb/loaders/hats/test_read_hats.py @@ -487,9 +487,14 @@ def test_read_hats_subset_with_order_search(small_sky_source_catalog, small_sky_ order_search = OrderSearch(min_order=1, max_order=2) # Filtering using catalog's order_search order_search_catalog = small_sky_source_catalog.order_search(min_order=1, max_order=2) + results1 = order_search_catalog.compute() # Filtering when calling `read_hats` - order_search_catalog_2 = lsdb.open_catalog(small_sky_source_dir, search_filter=order_search) + order_search_catalog_2 = lsdb.open_catalog( + small_sky_source_dir, search_filter=order_search, columns=["object_id", "object_ra", "object_dec"] + ) assert isinstance(order_search_catalog_2, lsdb.Catalog) + results2 = order_search_catalog_2.compute() + assert len(results1) == len(results2) # The partitions of the catalogs are equivalent assert order_search_catalog.get_healpix_pixels() == order_search_catalog_2.get_healpix_pixels()