Skip to content

Commit

Permalink
Replace some "point-cloud datasets" with "tile-based datasets"
Browse files Browse the repository at this point in the history
  • Loading branch information
olsen232 committed May 31, 2023
1 parent 1ba28df commit 295d880
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 9 deletions.
2 changes: 1 addition & 1 deletion kart/base_diff_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ def get_geometry_transforms(self, ds_path, ds_diff, context=None):
dataset = self._get_old_or_new_dataset(ds_path)
if dataset.DATASET_TYPE != "table":
# So far, table datasets are the only ones which have deltas transformed to the target CRS.
# TODO - support transformed output for point-cloud datasets too.
# TODO - support transformed output for tile-based datasets too.
return None, None

def _get_transform(source_crs):
Expand Down
2 changes: 1 addition & 1 deletion kart/diff_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def get_repo_diff(
to save repeated work.
repo_key_filter - controls which datasets (and PK values) match and are included in the diff.
convert_to_dataset_format - whether to show the diff of what would be committed if files were
converted to dataset format at commit-time (ie, for point-cloud tiles)
converted to dataset format at commit-time (ie, for point-cloud and raster tiles)
include_files - whether to include a DatasetDiff in the result for changes to files that
are simply standalone files, rather than part of a dataset's contents.
"""
Expand Down
13 changes: 8 additions & 5 deletions kart/lfs_commands/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from kart.repo import KartRepoState
from kart.structs import CommitWithReference
from kart.spatial_filter import SpatialFilter
from kart.tile import ALL_TILE_DATASET_TYPES

EMPTY_SHA = "0" * 40

Expand Down Expand Up @@ -100,7 +101,7 @@ def is_present(lfs_hash):
@click.argument("remote_url", required=False)
def pre_push(ctx, remote_name, remote_url, dry_run):
"""
Re-implementation of git-lfs pre-push - but, only searches for pointer blobs at **/.point-cloud-dataset.v?/tile/**
Re-implementation of git-lfs pre-push - but, only searches for pointer blobs at **/.<tile-based-dataset>/tile/**
(In contrast with git-lfs pre-push, which scans any and all blobs, looking for pointer files).
This means it won't encounter any features that are missing due to spatial filtering, which git-lfs stumbles over.
"""
Expand All @@ -124,8 +125,8 @@ def pre_push(ctx, remote_name, remote_url, dry_run):
repo, start_commits, [f"--remotes={remote_name}", *stop_commits]
):
# Because of the way a Kart repo is laid out, we know that:
# All LFS pointer files are blobs inside **/.point-cloud-dataset.v?/tile/**
# All blobs inside **/.point-cloud-dataset.v?/tile/** are LFS pointer files.
# All LFS pointer files are blobs inside **/.*-dataset.v?/tile/** and conversely,
# All blobs inside **/.*-dataset.v?/tile/** are LFS pointer files.
lfs_oids.add(get_hash_from_pointer_file(pointer_blob))

if dry_run:
Expand Down Expand Up @@ -273,7 +274,9 @@ def fetch_lfs_blobs_for_commits(

pointer_file_oids = set()
for commit in commits:
for dataset in repo.datasets(commit, filter_dataset_type="point-cloud"):
for dataset in repo.datasets(
commit, filter_dataset_type=ALL_TILE_DATASET_TYPES
):
pointer_file_oids.update(
blob.hex
for blob in dataset.tile_pointer_blobs(spatial_filter=spatial_filter)
Expand Down Expand Up @@ -391,7 +394,7 @@ def gc(ctx, dry_run):

spatial_filter = repo.spatial_filter
checked_out_lfs_oids = set()
for dataset in repo.datasets("HEAD", filter_dataset_type="point-cloud"):
for dataset in repo.datasets("HEAD", filter_dataset_type=ALL_TILE_DATASET_TYPES):
checked_out_lfs_oids.update(dataset.tile_lfs_hashes(spatial_filter))

to_delete = set()
Expand Down
2 changes: 1 addition & 1 deletion kart/rev_list_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def rev_list_feature_blobs(repo, start_commits, stop_commits):

def rev_list_tile_pointer_files(repo, start_commits, stop_commits):
"""
Yield all the blobs with a path identifying them as LFS pointers to the tiles of a point-cloud dataset.
Yield all the blobs with a path identifying them as LFS pointers to the tiles of a tile-based dataset.
Yields tuples in the form: (commit_id, match_result, blob).
To get the entire path, use match_result.group(0) - this can be decoded if necessary.
To get the dataset-path, use match_result.group(1)
Expand Down
2 changes: 1 addition & 1 deletion kart/spatial_filter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -719,7 +719,7 @@ def transform_for_tile_dataset(self, dataset):
"""
Transform this spatial filter so that it matches the CRS of the given dataset.
The resulting spatial filter will also have the extract_geometry function set such that
it can extract a flattened-to-2D point-cloud extent out of a tile summary.
it can extract a flattened-to-2D tile extent out of a tile summary.
"""
if self.match_all:
return SpatialFilter._MATCH_ALL
Expand Down

0 comments on commit 295d880

Please sign in to comment.