diff --git a/docs/rv2/index.rst b/docs/rv2/index.rst index 3ca506991..79187cae4 100644 --- a/docs/rv2/index.rst +++ b/docs/rv2/index.rst @@ -105,20 +105,21 @@ and maintain. ]) # Use the PyTorch backend for the SemanticSegmentation pipeline. - train_chip_sz = 300 + chip_sz = 300 backend = PyTorchSemanticSegmentationConfig( model=SemanticSegmentationModelConfig(backbone=Backbone.resnet50), solver=SolverConfig(lr=1e-4, num_epochs=1, batch_sz=2)) chip_options = SemanticSegmentationChipOptions( - window_method=SemanticSegmentationWindowMethod.random_sample, chips_per_scene=10) + window_method=SemanticSegmentationWindowMethod.random_sample, + chips_per_scene=10) return SemanticSegmentationConfig( root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=train_chip_sz, - chip_options=chip_options, - debug=False) + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz, + chip_options=chip_options) Raster Vision uses a ``unittest``-like method for executing pipelines. For instance, if the diff --git a/docs/rv2/quickstart.rst b/docs/rv2/quickstart.rst index e0f4ef2a9..f6109678b 100644 --- a/docs/rv2/quickstart.rst +++ b/docs/rv2/quickstart.rst @@ -108,21 +108,21 @@ Create a Python file in the ``${RV_QUICKSTART_CODE_DIR}`` named ``tiny_spacenet. ]) # Use the PyTorch backend for the SemanticSegmentation pipeline. - train_chip_sz = 300 + chip_sz = 300 backend = PyTorchSemanticSegmentationConfig( model=SemanticSegmentationModelConfig(backbone=Backbone.resnet50), solver=SolverConfig(lr=1e-4, num_epochs=1, batch_sz=2)) chip_options = SemanticSegmentationChipOptions( - window_method=SemanticSegmentationWindowMethod.random_sample, chips_per_scene=10) + window_method=SemanticSegmentationWindowMethod.random_sample, + chips_per_scene=10) return SemanticSegmentationConfig( root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=train_chip_sz, - chip_options=chip_options, - debug=False) - + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz, + chip_options=chip_options) Running the pipeline --------------------- diff --git a/rastervision2/core/analyzer/stats_analyzer_config.py b/rastervision2/core/analyzer/stats_analyzer_config.py index 53eba1cf9..776c62d72 100644 --- a/rastervision2/core/analyzer/stats_analyzer_config.py +++ b/rastervision2/core/analyzer/stats_analyzer_config.py @@ -12,11 +12,11 @@ class StatsAnalyzerConfig(AnalyzerConfig): description=( 'URI for output. If None and this is part of an RVPipeline, this is ' 'auto-generated.')) - sample_prob: float = Field( + sample_prob: Optional[float] = Field( 0.1, description=( - 'The probability of using a random window for computing statistics.' - )) + 'The probability of using a random window for computing statistics. ' + 'If None, will use a sliding window.')) def update(self, pipeline=None): if pipeline is not None and self.output_uri is None: diff --git a/rastervision2/core/data/label/semantic_segmentation_labels.py b/rastervision2/core/data/label/semantic_segmentation_labels.py index 491370138..72f765273 100644 --- a/rastervision2/core/data/label/semantic_segmentation_labels.py +++ b/rastervision2/core/data/label/semantic_segmentation_labels.py @@ -44,30 +44,20 @@ def set_label_arr(self, window, label_arr): def get_label_arr(self, window): return self.window_to_label_arr[window.tuple_format()] - def filter_by_aoi(self, aoi_polygons): - """Get the label array for a window. - - Args: - window: Box - extent: a Box representing the extent of the corresponding Scene - - Returns: - np.ndarray of class_ids with zeros filled in outside the AOIs and - clipped to the clip_extent - """ + def filter_by_aoi(self, aoi_polygons, null_class_id): new_labels = SemanticSegmentationLabels() for window in self.get_windows(): window_geom = window.to_shapely() label_arr = self.get_label_arr(window) - if not self.aoi_polygons: + if not aoi_polygons: return self else: # For each aoi_polygon, intersect with window, and put in window frame of # reference. window_aois = [] - for aoi in self.aoi_polygons: + for aoi in aoi_polygons: window_aoi = aoi.intersection(window_geom) if not window_aoi.is_empty: @@ -87,7 +77,7 @@ def transform_shape(x, y, z=None): out_shape=label_arr.shape, fill=1, dtype=np.uint8) - label_arr[mask.astype(np.bool)] = 0 + label_arr[mask.astype(np.bool)] = null_class_id new_labels.set_label_arr(window, label_arr) return new_labels diff --git a/rastervision2/core/data/label_source/object_detection_label_source.py b/rastervision2/core/data/label_source/object_detection_label_source.py index 9a1ff681e..5435ba5e8 100644 --- a/rastervision2/core/data/label_source/object_detection_label_source.py +++ b/rastervision2/core/data/label_source/object_detection_label_source.py @@ -3,7 +3,7 @@ class ObjectDetectionLabelSource(LabelSource): - def __init__(self, vector_source, extent): + def __init__(self, vector_source, extent=None): """Constructor. Args: diff --git a/rastervision2/core/data/raster_source/rasterized_source_config.py b/rastervision2/core/data/raster_source/rasterized_source_config.py index 8a6bec396..353a87099 100644 --- a/rastervision2/core/data/raster_source/rasterized_source_config.py +++ b/rastervision2/core/data/raster_source/rasterized_source_config.py @@ -1,6 +1,6 @@ from rastervision2.core.data.raster_source import (RasterizedSource) from rastervision2.core.data.vector_source import (VectorSourceConfig) -from rastervision2.pipeline.config import register_config, Config, Field +from rastervision2.pipeline.config import register_config, Config, Field, ConfigError @register_config('rasterizer') @@ -29,3 +29,9 @@ def build(self, class_config, crs_transformer, extent): return RasterizedSource(vector_source, self.rasterizer_config, extent, crs_transformer) + + def validate_config(self): + if self.vector_source.has_null_class_bufs(): + raise ConfigError( + 'Setting buffer to None for a class in the vector_source is ' + 'not allowed for RasterizedSourceConfig.') diff --git a/rastervision2/core/evaluation/semantic_segmentation_evaluation.py b/rastervision2/core/evaluation/semantic_segmentation_evaluation.py index 789c9a701..3b6df7132 100644 --- a/rastervision2/core/evaluation/semantic_segmentation_evaluation.py +++ b/rastervision2/core/evaluation/semantic_segmentation_evaluation.py @@ -138,7 +138,7 @@ def get_geoms(x): gt_count = len(gt) class_name = 'vector-{}-{}'.format( mode, - self.class_map.get_by_id(class_id).name) + self.class_config.names[class_id]) evaluation_item = ClassEvaluationItem(precision, recall, f1, count_error, gt_count, diff --git a/rastervision2/core/evaluation/semantic_segmentation_evaluator.py b/rastervision2/core/evaluation/semantic_segmentation_evaluator.py index 152959bcb..7f8153ef1 100644 --- a/rastervision2/core/evaluation/semantic_segmentation_evaluator.py +++ b/rastervision2/core/evaluation/semantic_segmentation_evaluator.py @@ -4,7 +4,7 @@ from shapely.strtree import STRtree from rastervision2.core.data import ActivateMixin -from rastervision2.core.data.vector_source import GeoJSONVectorSource +from rastervision2.core.data.vector_source import GeoJSONVectorSourceConfig from rastervision2.core.evaluation import (ClassificationEvaluator, SemanticSegmentationEvaluation) @@ -44,6 +44,7 @@ def create_evaluation(self): def process(self, scenes, tmp_dir): evaluation = self.create_evaluation() vect_evaluation = self.create_evaluation() + null_class_id = self.class_config.get_null_class_id() for scene in scenes: log.info('Computing evaluation for scene {}...'.format(scene.id)) @@ -56,8 +57,9 @@ def process(self, scenes, tmp_dir): if scene.aoi_polygons: # Filter labels based on AOI. ground_truth = ground_truth.filter_by_aoi( - scene.aoi_polygons) - predictions = predictions.filter_by_aoi(scene.aoi_polygons) + scene.aoi_polygons, null_class_id) + predictions = predictions.filter_by_aoi( + scene.aoi_polygons, null_class_id) scene_evaluation = self.create_evaluation() scene_evaluation.compute(ground_truth, predictions) evaluation.merge(scene_evaluation, scene_id=scene.id) @@ -69,10 +71,11 @@ def process(self, scenes, tmp_dir): ) for vo in label_store.vector_output: pred_geojson_uri = vo.uri - mode = vo.mode + mode = vo.get_mode() class_id = vo.class_id - pred_geojson_source = GeoJSONVectorSource( - pred_geojson_uri, + pred_geojson_source = GeoJSONVectorSourceConfig( + uri=pred_geojson_uri, default_class_id=None).build( + self.class_config, scene.raster_source.get_crs_transformer()) pred_geojson = pred_geojson_source.get_geojson() diff --git a/rastervision2/core/rv_pipeline/rv_pipeline_config.py b/rastervision2/core/rv_pipeline/rv_pipeline_config.py index 1f5e7d4e5..a3c14b664 100644 --- a/rastervision2/core/rv_pipeline/rv_pipeline_config.py +++ b/rastervision2/core/rv_pipeline/rv_pipeline_config.py @@ -34,11 +34,10 @@ class RVPipelineConfig(PipelineConfig): ('Analyzers to run during analyzer command. A StatsAnalyzer will be added ' 'automatically if any scenes have a RasterTransformer.')) - debug: bool = Field(False, description='If True, use debug mode.') train_chip_sz: int = Field( - 200, description='Size of training chips in pixels.') + 300, description='Size of training chips in pixels.') predict_chip_sz: int = Field( - 200, description='Size of predictions chips in pixels.') + 300, description='Size of predictions chips in pixels.') predict_batch_sz: int = Field( 8, description='Batch size to use during prediction.') diff --git a/rastervision2/examples/chip_classification.py b/rastervision2/examples/chip_classification.py index e2de229e9..dcb535fbe 100644 --- a/rastervision2/examples/chip_classification.py +++ b/rastervision2/examples/chip_classification.py @@ -78,6 +78,7 @@ def make_scene(scene_info): label_source=label_source, aoi_uris=[aoi_uri]) + chip_sz = 200 train_scenes = [make_scene(info) for info in train_scene_info] val_scenes = [make_scene(info) for info in val_scene_info] dataset = DatasetConfig( @@ -92,12 +93,13 @@ def make_scene(scene_info): model=model, solver=solver, log_tensorboard=log_tensorboard, - run_tensorboard=run_tensorboard) + run_tensorboard=run_tensorboard, + test_mode=test) config = ChipClassificationConfig( root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=200, - debug=debug) + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz) return config diff --git a/rastervision2/examples/object_detection.py b/rastervision2/examples/object_detection.py index deeb01a7c..44468d18b 100644 --- a/rastervision2/examples/object_detection.py +++ b/rastervision2/examples/object_detection.py @@ -61,6 +61,7 @@ def make_scene(id): id=id, raster_source=raster_source, label_source=label_source) class_config = ClassConfig(names=['vehicle']) + chip_sz = 300 dataset = DatasetConfig( class_config=class_config, train_scenes=[make_scene(id) for id in train_ids], @@ -78,13 +79,14 @@ def make_scene(id): batch_sz=16, one_cycle=True), log_tensorboard=True, - run_tensorboard=False) + run_tensorboard=False, + test_mode=test) return ObjectDetectionConfig( root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=300, + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz, chip_options=chip_options, - predict_options=predict_options, - debug=test) + predict_options=predict_options) diff --git a/rastervision2/examples/semantic_segmentation.py b/rastervision2/examples/semantic_segmentation.py index cc2a640cd..a9fc1a0f9 100644 --- a/rastervision2/examples/semantic_segmentation.py +++ b/rastervision2/examples/semantic_segmentation.py @@ -94,9 +94,9 @@ def make_scene(id): class_config=class_config, train_scenes=[make_scene(id) for id in train_ids], validation_scenes=[make_scene(id) for id in val_ids]) - train_chip_sz = 300 + chip_sz = 300 chip_options = SemanticSegmentationChipOptions( - window_method=SemanticSegmentationWindowMethod.sliding, stride=300) + window_method=SemanticSegmentationWindowMethod.sliding, stride=chip_sz) backend = PyTorchSemanticSegmentationConfig( model=SemanticSegmentationModelConfig(backbone=Backbone.resnet50), @@ -107,12 +107,13 @@ def make_scene(id): batch_sz=8, one_cycle=True), log_tensorboard=True, - run_tensorboard=False) + run_tensorboard=False, + test_mode=test) return SemanticSegmentationConfig( root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=train_chip_sz, - chip_options=chip_options, - debug=test) + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz, + chip_options=chip_options) diff --git a/rastervision2/examples/tiny_spacenet.py b/rastervision2/examples/tiny_spacenet.py index 2065987dd..b6aaec8ea 100644 --- a/rastervision2/examples/tiny_spacenet.py +++ b/rastervision2/examples/tiny_spacenet.py @@ -58,7 +58,7 @@ def make_scene(scene_id, image_uri, label_uri): ]) # Use the PyTorch backend for the SemanticSegmentation pipeline. - train_chip_sz = 300 + chip_sz = 300 backend = PyTorchSemanticSegmentationConfig( model=SemanticSegmentationModelConfig(backbone=Backbone.resnet50), solver=SolverConfig(lr=1e-4, num_epochs=1, batch_sz=2)) @@ -70,6 +70,6 @@ def make_scene(scene_id, image_uri, label_uri): root_uri=root_uri, dataset=dataset, backend=backend, - train_chip_sz=train_chip_sz, - chip_options=chip_options, - debug=False) + train_chip_sz=chip_sz, + predict_chip_sz=chip_sz, + chip_options=chip_options) diff --git a/rastervision2/pytorch_backend/pytorch_chip_classification_config.py b/rastervision2/pytorch_backend/pytorch_chip_classification_config.py index fe2c361f7..cf9936f83 100644 --- a/rastervision2/pytorch_backend/pytorch_chip_classification_config.py +++ b/rastervision2/pytorch_backend/pytorch_chip_classification_config.py @@ -25,7 +25,7 @@ def get_learner_config(self, pipeline): data=data, model=self.model, solver=self.solver, - test_mode=pipeline.debug, + test_mode=self.test_mode, output_uri=pipeline.train_uri, log_tensorboard=self.log_tensorboard, run_tensorboard=self.run_tensorboard) diff --git a/rastervision2/pytorch_backend/pytorch_learner_backend_config.py b/rastervision2/pytorch_backend/pytorch_learner_backend_config.py index bb2b0a370..3185a2a6d 100644 --- a/rastervision2/pytorch_backend/pytorch_learner_backend_config.py +++ b/rastervision2/pytorch_backend/pytorch_learner_backend_config.py @@ -21,6 +21,12 @@ class PyTorchLearnerBackendConfig(BackendConfig): description=( 'Names of albumentations augmentors to use for training batches. ' 'Choices include: ' + str(augmentor_list))) + test_mode: bool = Field( + False, + description= + ('This field is passed along to the LearnerConfig which is returned by ' + 'get_learner_config(). For more info, see the docs for' + 'pytorch_learner.learner_config.LearnerConfig.test_mode.')) def get_bundle_filenames(self): return ['model-bundle.zip'] diff --git a/rastervision2/pytorch_backend/pytorch_object_detection_config.py b/rastervision2/pytorch_backend/pytorch_object_detection_config.py index de525baf1..58f82540f 100644 --- a/rastervision2/pytorch_backend/pytorch_object_detection_config.py +++ b/rastervision2/pytorch_backend/pytorch_object_detection_config.py @@ -24,7 +24,7 @@ def get_learner_config(self, pipeline): data=data, model=self.model, solver=self.solver, - test_mode=pipeline.debug, + test_mode=self.test_mode, output_uri=pipeline.train_uri, log_tensorboard=self.log_tensorboard, run_tensorboard=self.run_tensorboard) diff --git a/rastervision2/pytorch_backend/pytorch_semantic_segmentation_config.py b/rastervision2/pytorch_backend/pytorch_semantic_segmentation_config.py index 1409f11d5..6d35bb4be 100644 --- a/rastervision2/pytorch_backend/pytorch_semantic_segmentation_config.py +++ b/rastervision2/pytorch_backend/pytorch_semantic_segmentation_config.py @@ -24,7 +24,7 @@ def get_learner_config(self, pipeline): data=data, model=self.model, solver=self.solver, - test_mode=pipeline.debug, + test_mode=self.test_mode, output_uri=pipeline.train_uri, log_tensorboard=self.log_tensorboard, run_tensorboard=self.run_tensorboard) diff --git a/tests_v2/__init__.py b/tests_v2/__init__.py new file mode 100644 index 000000000..abe012204 --- /dev/null +++ b/tests_v2/__init__.py @@ -0,0 +1,6 @@ +import os + + +def data_file_path(rel_path): + data_dir = os.path.join(os.path.dirname(__file__), 'data_files') + return os.path.join(data_dir, rel_path) diff --git a/test_v2/__init__.py b/tests_v2/core/__init__.py similarity index 100% rename from test_v2/__init__.py rename to tests_v2/core/__init__.py diff --git a/test_v2/pipeline/__init__.py b/tests_v2/core/data/__init__.py similarity index 100% rename from test_v2/pipeline/__init__.py rename to tests_v2/core/data/__init__.py diff --git a/tests_v2/core/data/label/__init__.py b/tests_v2/core/data/label/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/data/label/test_chip_classification_labels.py b/tests_v2/core/data/label/test_chip_classification_labels.py new file mode 100644 index 000000000..d0dfdfbb0 --- /dev/null +++ b/tests_v2/core/data/label/test_chip_classification_labels.py @@ -0,0 +1,85 @@ +import unittest + +from rastervision2.core.box import Box +from rastervision2.core.data.label.chip_classification_labels import ( + ChipClassificationLabels) + + +class TestChipClassificationLabels(unittest.TestCase): + def setUp(self): + self.labels = ChipClassificationLabels() + + self.cell1 = Box.make_square(0, 0, 2) + self.class_id1 = 1 + self.labels.set_cell(self.cell1, self.class_id1) + + self.cell2 = Box.make_square(0, 2, 2) + self.class_id2 = 2 + self.labels.set_cell(self.cell2, self.class_id2) + + def test_get_cell(self): + cell = Box.make_square(0, 2, 3) + class_id = self.labels.get_cell_class_id(cell) + self.assertEqual(class_id, None) + + class_id = self.labels.get_cell_class_id(self.cell1) + self.assertEqual(class_id, self.class_id1) + + class_id = self.labels.get_cell_class_id(self.cell2) + self.assertEqual(class_id, self.class_id2) + + def test_get_singleton_labels(self): + labels = self.labels.get_singleton_labels(self.cell1) + + cells = labels.get_cells() + self.assertEqual(len(cells), 1) + + class_id = labels.get_cell_class_id(self.cell1) + self.assertEqual(class_id, self.class_id1) + + def test_get_cells(self): + cells = self.labels.get_cells() + self.assertEqual(len(cells), 2) + # ordering of cells isn't known + self.assertTrue((cells[0] == self.cell1 and cells[1] == self.cell2) + or (cells[1] == self.cell1 and cells[0] == self.cell2)) + + def test_get_class_ids(self): + cells = self.labels.get_cells() + class_ids = self.labels.get_class_ids() + # check that order of class_ids corresponds to order of cells + if (cells[0] == self.cell1 and cells[1] == self.cell2): + self.assertListEqual(class_ids, [1, 2]) + elif (cells[1] == self.cell1 and cells[0] == self.cell2): + self.assertListEqual(class_ids, [2, 1]) + + def test_extend(self): + labels = ChipClassificationLabels() + cell3 = Box.make_square(0, 4, 2) + class_id3 = 1 + labels.set_cell(cell3, class_id3) + + self.labels.extend(labels) + cells = self.labels.get_cells() + self.assertEqual(len(cells), 3) + self.assertTrue(cell3 in cells) + + def test_filter_by_aoi(self): + aois = [Box.make_square(0, 0, 2).to_shapely()] + filt_labels = self.labels.filter_by_aoi(aois) + + exp_labels = ChipClassificationLabels() + cell1 = Box.make_square(0, 0, 2) + class_id1 = 1 + exp_labels.set_cell(cell1, class_id1) + self.assertEqual(filt_labels, exp_labels) + + aois = [Box.make_square(4, 4, 2).to_shapely()] + filt_labels = self.labels.filter_by_aoi(aois) + + exp_labels = ChipClassificationLabels() + self.assertEqual(filt_labels, exp_labels) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label/test_object_detection_labels.py b/tests_v2/core/data/label/test_object_detection_labels.py new file mode 100644 index 000000000..de7f78f16 --- /dev/null +++ b/tests_v2/core/data/label/test_object_detection_labels.py @@ -0,0 +1,200 @@ +import unittest + +import numpy as np + +from rastervision2.core.box import Box +from rastervision2.core.data.class_config import ClassConfig +from rastervision2.core.data.label.object_detection_labels import ( + ObjectDetectionLabels) +from rastervision2.core.data.label.tfod_utils.np_box_list import BoxList + + +class ObjectDetectionLabelsTest(unittest.TestCase): + def setUp(self): + self.class_config = ClassConfig(names=['car', 'house']) + self.npboxes = np.array([ + [0., 0., 2., 2.], + [2., 2., 4., 4.], + ]) + self.class_ids = np.array([0, 1]) + self.scores = np.array([0.9, 0.9]) + self.labels = ObjectDetectionLabels( + self.npboxes, self.class_ids, scores=self.scores) + + def test_from_boxlist(self): + boxlist = BoxList(self.npboxes) + boxlist.add_field('classes', self.class_ids) + boxlist.add_field('scores', self.scores) + labels = ObjectDetectionLabels.from_boxlist(boxlist) + labels.assert_equal(self.labels) + + def test_make_empty(self): + npboxes = np.empty((0, 4)) + class_ids = np.empty((0, )) + scores = np.empty((0, )) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + + labels = ObjectDetectionLabels.make_empty() + labels.assert_equal(expected_labels) + + def test_constructor(self): + labels = ObjectDetectionLabels( + self.npboxes, self.class_ids, scores=self.scores) + expected_labels = ObjectDetectionLabels(self.npboxes, self.class_ids, + self.scores) + labels.assert_equal(expected_labels) + + labels = ObjectDetectionLabels(self.npboxes, self.class_ids) + scores = np.ones(self.class_ids.shape) + expected_labels = ObjectDetectionLabels( + self.npboxes, self.class_ids, scores=scores) + labels.assert_equal(expected_labels) + + def test_get_boxes(self): + boxes = self.labels.get_boxes() + self.assertEqual(len(boxes), 2) + np.testing.assert_array_equal(boxes[0].npbox_format(), + self.npboxes[0, :]) + np.testing.assert_array_equal(boxes[1].npbox_format(), + self.npboxes[1, :]) + + def test_len(self): + nb_labels = len(self.labels) + self.assertEqual(self.npboxes.shape[0], nb_labels) + + def test_local_to_global(self): + local_npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.]]) + window = Box.make_square(10, 10, 10) + global_npboxes = ObjectDetectionLabels.local_to_global( + local_npboxes, window) + + expected_global_npboxes = np.array([[10., 10., 12., 12.], + [12., 12., 14., 14.]]) + np.testing.assert_array_equal(global_npboxes, expected_global_npboxes) + + def test_global_to_local(self): + global_npboxes = np.array([[10., 10., 12., 12.], [12., 12., 14., 14.]]) + window = Box.make_square(10, 10, 10) + local_npboxes = ObjectDetectionLabels.global_to_local( + global_npboxes, window) + + expected_local_npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.]]) + np.testing.assert_array_equal(local_npboxes, expected_local_npboxes) + + def test_local_to_normalized(self): + local_npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.]]) + window = Box(0, 0, 10, 100) + norm_npboxes = ObjectDetectionLabels.local_to_normalized( + local_npboxes, window) + + expected_norm_npboxes = np.array([[0., 0., 0.2, 0.02], + [0.2, 0.02, 0.4, 0.04]]) + np.testing.assert_array_equal(norm_npboxes, expected_norm_npboxes) + + def test_normalized_to_local(self): + norm_npboxes = np.array([[0., 0., 0.2, 0.02], [0.2, 0.02, 0.4, 0.04]]) + window = Box(0, 0, 10, 100) + local_npboxes = ObjectDetectionLabels.normalized_to_local( + norm_npboxes, window) + + expected_local_npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.]]) + np.testing.assert_array_equal(local_npboxes, expected_local_npboxes) + + def test_get_overlapping(self): + window = Box.make_square(0, 0, 2.01) + labels = ObjectDetectionLabels.get_overlapping(self.labels, window) + labels.assert_equal(self.labels) + + window = Box.make_square(0, 0, 3) + labels = ObjectDetectionLabels.get_overlapping( + self.labels, window, ioa_thresh=0.5) + npboxes = np.array([[0., 0., 2., 2.]]) + class_ids = np.array([0]) + scores = np.array([0.9]) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + labels.assert_equal(expected_labels) + + window = Box.make_square(0, 0, 3) + labels = ObjectDetectionLabels.get_overlapping( + self.labels, window, ioa_thresh=0.1, clip=True) + expected_npboxes = np.array([ + [0., 0., 2., 2.], + [2., 2., 3., 3.], + ]) + expected_labels = ObjectDetectionLabels( + expected_npboxes, self.class_ids, scores=self.scores) + labels.assert_equal(expected_labels) + + def test_concatenate(self): + npboxes = np.array([[4., 4., 5., 5.]]) + class_ids = np.array([1]) + scores = np.array([0.3]) + labels = ObjectDetectionLabels(npboxes, class_ids, scores=scores) + new_labels = ObjectDetectionLabels.concatenate(self.labels, labels) + + npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.], + [4., 4., 5., 5.]]) + class_ids = np.array([0, 1, 1]) + scores = np.array([0.9, 0.9, 0.3]) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + new_labels.assert_equal(expected_labels) + + def test_prune_duplicates(self): + # This first box has a score below score_thresh so it should get + # pruned. The third box overlaps with the second, but has higher score, + # so the second one should get pruned. The fourth box overlaps with + # the second less than merge_thresh, so it should not get pruned. + npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.], + [2.1, 2.1, 4.1, 4.1], [3.5, 3.5, 5.5, 5.5]]) + class_ids = np.array([0, 1, 0, 1]) + scores = np.array([0.2, 0.9, 0.9, 1.0]) + labels = ObjectDetectionLabels(npboxes, class_ids, scores=scores) + score_thresh = 0.5 + merge_thresh = 0.5 + pruned_labels = ObjectDetectionLabels.prune_duplicates( + labels, score_thresh, merge_thresh) + + self.assertEqual(len(pruned_labels), 2) + + expected_npboxes = np.array([[2.1, 2.1, 4.1, 4.1], + [3.5, 3.5, 5.5, 5.5]]) + expected_class_ids = np.array([0, 1]) + expected_scores = np.array([0.9, 1.0]) + + # prune_duplicates does not maintain ordering of boxes, so find match + # between pruned boxes and expected_npboxes. + pruned_npboxes = pruned_labels.get_npboxes() + pruned_inds = [None, None] + for box_ind, box in enumerate(expected_npboxes): + for pruned_box_ind, pruned_box in enumerate(pruned_npboxes): + if np.array_equal(pruned_box, box): + pruned_inds[box_ind] = pruned_box_ind + self.assertTrue(np.all(pruned_inds is not None)) + + expected_labels = ObjectDetectionLabels( + expected_npboxes[pruned_inds], + expected_class_ids[pruned_inds], + scores=expected_scores[pruned_inds]) + pruned_labels.assert_equal(expected_labels) + + def test_filter_by_aoi(self): + aois = [Box.make_square(0, 0, 2).to_shapely()] + filt_labels = self.labels.filter_by_aoi(aois) + + npboxes = np.array([[0., 0., 2., 2.]]) + class_ids = np.array([0]) + scores = np.array([0.9]) + exp_labels = ObjectDetectionLabels(npboxes, class_ids, scores=scores) + self.assertEqual(filt_labels, exp_labels) + + aois = [Box.make_square(4, 4, 2).to_shapely()] + filt_labels = self.labels.filter_by_aoi(aois) + exp_labels = ObjectDetectionLabels.make_empty() + self.assertEqual(filt_labels, exp_labels) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label/test_semantic_segmentation_labels.py b/tests_v2/core/data/label/test_semantic_segmentation_labels.py new file mode 100644 index 000000000..d0265132f --- /dev/null +++ b/tests_v2/core/data/label/test_semantic_segmentation_labels.py @@ -0,0 +1,36 @@ +import unittest + +import numpy as np + +from rastervision2.core.box import Box +from rastervision2.core.data.label import SemanticSegmentationLabels + + +class TestSemanticSegmentationLabels(unittest.TestCase): + def setUp(self): + self.windows = [Box.make_square(0, 0, 10), Box.make_square(0, 10, 10)] + self.label_arr0 = np.random.choice([0, 1], (10, 10)) + self.label_arr1 = np.random.choice([0, 1], (10, 10)) + self.labels = SemanticSegmentationLabels() + self.labels.set_label_arr(self.windows[0], self.label_arr0) + self.labels.set_label_arr(self.windows[1], self.label_arr1) + + def test_get(self): + np.testing.assert_array_equal( + self.labels.get_label_arr(self.windows[0]), self.label_arr0) + + def test_get_with_aoi(self): + null_class_id = 2 + + aoi_polygons = [Box.make_square(5, 15, 2).to_shapely()] + exp_label_arr = np.full(self.label_arr1.shape, null_class_id) + exp_label_arr[5:7, 5:7] = self.label_arr1[5:7, 5:7] + + labels = self.labels.filter_by_aoi(aoi_polygons, null_class_id) + label_arr = labels.get_label_arr(self.windows[1]) + np.testing.assert_array_equal(label_arr, exp_label_arr) + self.assertEqual(1, len(labels.window_to_label_arr)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label_source/__init__.py b/tests_v2/core/data/label_source/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/data/label_source/test_chip_classification_label_source.py b/tests_v2/core/data/label_source/test_chip_classification_label_source.py new file mode 100644 index 000000000..cb8efe37d --- /dev/null +++ b/tests_v2/core/data/label_source/test_chip_classification_label_source.py @@ -0,0 +1,247 @@ +import unittest +import os + +from shapely.geometry import shape +from shapely.strtree import STRtree + +from rastervision2.pipeline import rv_config +from rastervision2.pipeline.file_system import json_to_file +from rastervision2.core.box import Box +from rastervision2.core.data import ( + ClassConfig, infer_cell, ChipClassificationLabelSourceConfig, + GeoJSONVectorSourceConfig) + +from tests_v2.core.data.mock_crs_transformer import DoubleCRSTransformer + + +class TestChipClassificationLabelSource(unittest.TestCase): + def setUp(self): + self.crs_transformer = DoubleCRSTransformer() + self.geojson = { + 'type': + 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'geometry': { + 'type': + 'MultiPolygon', + 'coordinates': [[[[0., 0.], [0., 2.], [2., 2.], [2., 0.], + [0., 0.]]]] + }, + 'properties': { + 'class_name': 'car', + 'class_id': 0, + 'score': 0.0 + } + }, { + 'type': 'Feature', + 'geometry': { + 'type': + 'Polygon', + 'coordinates': [[[2., 2.], [2., 4.], [4., 4.], [4., 2.], + [2., 2.]]] + }, + 'properties': { + 'score': 0.0, + 'class_name': 'house', + 'class_id': 1 + } + }] + } + + self.class_config = ClassConfig(names=['car', 'house']) + + self.box1 = Box.make_square(0, 0, 4) + self.box2 = Box.make_square(4, 4, 4) + self.class_id1 = 0 + self.class_id2 = 1 + self.background_class_id = 2 + + geoms = [] + for f in self.geojson['features']: + g = shape(f['geometry']) + g.class_id = f['properties']['class_id'] + geoms.append(g) + self.str_tree = STRtree(geoms) + + self.file_name = 'labels.json' + self.tmp_dir = rv_config.get_tmp_dir() + self.uri = os.path.join(self.tmp_dir.name, self.file_name) + json_to_file(self.geojson, self.uri) + + def tearDown(self): + self.tmp_dir.cleanup() + + def test_infer_cell1(self): + # More of box 1 is in cell. + cell = Box.make_square(0, 0, 3) + ioa_thresh = 0.5 + use_intersection_over_cell = False + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, self.class_id1) + + def test_infer_cell2(self): + # More of box 2 is in cell. + cell = Box.make_square(1, 1, 3) + ioa_thresh = 0.5 + use_intersection_over_cell = False + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, self.class_id2) + + def test_infer_cell3(self): + # Only box 2 is in cell, but IOA isn't high enough. + cell = Box.make_square(3, 3, 3) + ioa_thresh = 0.5 + use_intersection_over_cell = False + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, None) + + def test_infer_cell4(self): + # Both boxes inside cell, but using intersection_over_cell, + # the IOA isn't high enough. + cell = Box.make_square(0, 0, 10) + ioa_thresh = 0.5 + use_intersection_over_cell = True + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, None) + + def test_infer_cell5(self): + # More of box1 in cell, using intersection_over_cell with the + # IOA high enough. + cell = Box.make_square(0, 0, 3) + ioa_thresh = 0.4 + use_intersection_over_cell = True + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, self.class_id1) + + def test_infer_cell6(self): + # No boxes overlap enough, use background_class_id + cell = Box.make_square(0, 0, 10) + ioa_thresh = 0.5 + use_intersection_over_cell = True + background_class_id = self.background_class_id + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, self.background_class_id) + + def test_infer_cell7(self): + # Cell doesn't overlap with any boxes. + cell = Box.make_square(10, 10, 1) + ioa_thresh = 0.5 + use_intersection_over_cell = True + background_class_id = None + pick_min_class_id = False + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, None) + + def test_infer_cell8(self): + # box2 overlaps more than box1, but using pick_min_class_id, so + # picks box1. + cell = Box.make_square(1, 1, 3) + ioa_thresh = 0.5 + use_intersection_over_cell = False + background_class_id = None + pick_min_class_id = True + + class_id = infer_cell(cell, self.str_tree, ioa_thresh, + use_intersection_over_cell, background_class_id, + pick_min_class_id) + self.assertEqual(class_id, self.class_id2) + + def test_get_labels_inferred(self): + extent = Box.make_square(0, 0, 8) + + config = ChipClassificationLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.uri, default_class_id=None), + ioa_thresh=0.5, + use_intersection_over_cell=False, + pick_min_class_id=False, + background_class_id=self.background_class_id, + infer_cells=True, + cell_sz=4) + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + cells = labels.get_cells() + + self.assertEqual(len(cells), 4) + self.assertEqual(labels.get_cell_class_id(self.box1), self.class_id1) + self.assertEqual(labels.get_cell_class_id(self.box2), self.class_id2) + self.assertEqual( + labels.get_cell_class_id(Box.make_square(0, 4, 4)), + self.background_class_id) + self.assertEqual( + labels.get_cell_class_id(Box.make_square(4, 0, 4)), + self.background_class_id) + + def test_get_labels_small_extent(self): + # Extent only has enough of first box in it. + extent = Box.make_square(0, 0, 2) + + config = ChipClassificationLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.uri, default_class_id=None)) + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + + cells = labels.get_cells() + self.assertEqual(len(cells), 1) + class_id = labels.get_cell_class_id(self.box1) + self.assertEqual(class_id, self.class_id1) + class_id = labels.get_cell_class_id(self.box2) + self.assertEqual(class_id, None) + + def test_get_labels(self): + # Extent contains both boxes. + extent = Box.make_square(0, 0, 8) + + config = ChipClassificationLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.uri, default_class_id=None)) + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + + cells = labels.get_cells() + self.assertEqual(len(cells), 2) + class_id = labels.get_cell_class_id(self.box1) + self.assertEqual(class_id, self.class_id1) + class_id = labels.get_cell_class_id(self.box2) + self.assertEqual(class_id, self.class_id2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label_source/test_object_detection_label_source.py b/tests_v2/core/data/label_source/test_object_detection_label_source.py new file mode 100644 index 000000000..0865de30b --- /dev/null +++ b/tests_v2/core/data/label_source/test_object_detection_label_source.py @@ -0,0 +1,111 @@ +import unittest +import os + +import numpy as np + +from rastervision2.core.data import ( + ObjectDetectionLabelSourceConfig, GeoJSONVectorSourceConfig, ObjectDetectionLabels, + ClassConfig) +from rastervision2.core import Box +from rastervision2.pipeline import rv_config +from rastervision2.pipeline.file_system import json_to_file + +from tests_v2.core.data.mock_crs_transformer import DoubleCRSTransformer + + +class TestObjectDetectionLabelSource(unittest.TestCase): + def setUp(self): + self.file_name = 'labels.json' + self.tmp_dir = rv_config.get_tmp_dir() + self.file_path = os.path.join(self.tmp_dir.name, self.file_name) + + self.crs_transformer = DoubleCRSTransformer() + self.geojson = { + 'type': + 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'geometry': { + 'type': + 'Polygon', + 'coordinates': [[[0., 0.], [0., 1.], [1., 1.], [1., 0.], + [0., 0.]]] + }, + 'properties': { + 'class_id': 0, + 'score': 0.9 + } + }, { + 'type': 'Feature', + 'geometry': { + 'type': + 'Polygon', + 'coordinates': [[[1., 1.], [1., 2.], [2., 2.], [2., 1.], + [1., 1.]]] + }, + 'properties': { + 'score': 0.9, + 'class_id': 1 + } + }] + } + + self.extent = Box.make_square(0, 0, 10) + self.class_config = ClassConfig(names=['car', 'house']) + json_to_file(self.geojson, self.file_path) + + def tearDown(self): + self.tmp_dir.cleanup() + + def test_read_without_extent(self): + config = ObjectDetectionLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.file_path, default_class_id=None)) + extent = None + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + + npboxes = np.array([[0., 0., 2., 2.], [2., 2., 4., 4.]]) + class_ids = np.array([0, 1]) + scores = np.array([0.9, 0.9]) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + labels.assert_equal(expected_labels) + + def test_read_with_extent(self): + # Extent only includes the first box. + extent = Box.make_square(0, 0, 3) + config = ObjectDetectionLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.file_path, default_class_id=None)) + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + + npboxes = np.array([[0., 0., 2., 2.]]) + class_ids = np.array([0]) + scores = np.array([0.9]) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + labels.assert_equal(expected_labels) + + # Extent includes both boxes, but clips the second. + extent = Box.make_square(0, 0, 3.9) + config = ObjectDetectionLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=self.file_path, default_class_id=None)) + source = config.build( + self.class_config, self.crs_transformer, extent, self.tmp_dir.name) + labels = source.get_labels() + + npboxes = np.array([[0., 0., 2., 2.], [2., 2., 3.9, 3.9]]) + class_ids = np.array([0, 1]) + scores = np.array([0.9, 0.9]) + expected_labels = ObjectDetectionLabels( + npboxes, class_ids, scores=scores) + labels.assert_equal(expected_labels) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label_source/test_segmentation_class_transformer.py b/tests_v2/core/data/label_source/test_segmentation_class_transformer.py new file mode 100644 index 000000000..faa559a2d --- /dev/null +++ b/tests_v2/core/data/label_source/test_segmentation_class_transformer.py @@ -0,0 +1,36 @@ +import unittest + +import numpy as np + +from rastervision2.core.data import SegmentationClassTransformer +from rastervision2.core.data.utils import color_to_triple +from rastervision2.core.data.class_config import ClassConfig + + +class TestSegmentationClassTransformer(unittest.TestCase): + def setUp(self): + self.class_config = ClassConfig( + names=['a', 'b', 'c'], colors=['red', 'green', 'blue']) + self.class_config.ensure_null_class() + self.transformer = SegmentationClassTransformer(self.class_config) + + self.rgb_image = np.zeros((1, 3, 3)) + self.rgb_image[0, 0, :] = color_to_triple('red') + self.rgb_image[0, 1, :] = color_to_triple('green') + self.rgb_image[0, 2, :] = color_to_triple('blue') + + self.class_image = np.array([[0, 1, 2]]) + + def test_rgb_to_class(self): + class_image = self.transformer.rgb_to_class(self.rgb_image) + expected_class_image = self.class_image + np.testing.assert_array_equal(class_image, expected_class_image) + + def test_class_to_rgb(self): + rgb_image = self.transformer.class_to_rgb(self.class_image) + expected_rgb_image = self.rgb_image + np.testing.assert_array_equal(rgb_image, expected_rgb_image) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/label_source/test_semantic_segmentation_label_source.py b/tests_v2/core/data/label_source/test_semantic_segmentation_label_source.py new file mode 100644 index 000000000..840e7d31e --- /dev/null +++ b/tests_v2/core/data/label_source/test_semantic_segmentation_label_source.py @@ -0,0 +1,63 @@ +import unittest + +import numpy as np + +from rastervision2.core import Box +from rastervision2.core.data import ClassConfig, SemanticSegmentationLabelSource +from tests_v2.core.data.mock_raster_source import MockRasterSource + + +class TestSemanticSegmentationLabelSource(unittest.TestCase): + def test_enough_target_pixels_true(self): + data = np.zeros((10, 10, 1), dtype=np.uint8) + data[4:, 4:, :] = 1 + raster_source = MockRasterSource([0], 1) + raster_source.set_raster(data) + label_source = SemanticSegmentationLabelSource(raster_source=raster_source) + with label_source.activate(): + extent = Box(0, 0, 10, 10) + self.assertTrue(label_source.enough_target_pixels(extent, 30, [1])) + + def test_enough_target_pixels_false(self): + data = np.zeros((10, 10, 1), dtype=np.uint8) + data[7:, 7:, :] = 1 + raster_source = MockRasterSource([0], 1) + raster_source.set_raster(data) + label_source = SemanticSegmentationLabelSource(raster_source=raster_source) + with label_source.activate(): + extent = Box(0, 0, 10, 10) + self.assertFalse( + label_source.enough_target_pixels(extent, 30, [1])) + + def test_get_labels(self): + data = np.zeros((10, 10, 1), dtype=np.uint8) + data[7:, 7:, 0] = 1 + raster_source = MockRasterSource([0], 1) + raster_source.set_raster(data) + label_source = SemanticSegmentationLabelSource(raster_source=raster_source) + with label_source.activate(): + window = Box.make_square(7, 7, 3) + labels = label_source.get_labels(window=window) + label_arr = labels.get_label_arr(window) + expected_label_arr = np.ones((3, 3)) + np.testing.assert_array_equal(label_arr, expected_label_arr) + + def test_get_labels_rgb(self): + data = np.zeros((10, 10, 3), dtype=np.uint8) + data[7:, 7:, :] = [1, 1, 1] + raster_source = MockRasterSource([0, 1, 2], 3) + raster_source.set_raster(data) + rgb_class_config = ClassConfig(names=['a'], colors=['#010101']) + rgb_class_config.ensure_null_class() + label_source = SemanticSegmentationLabelSource( + raster_source=raster_source, rgb_class_config=rgb_class_config) + with label_source.activate(): + window = Box.make_square(7, 7, 3) + labels = label_source.get_labels(window=window) + label_arr = labels.get_label_arr(window) + expected_label_arr = np.zeros((3, 3)) + np.testing.assert_array_equal(label_arr, expected_label_arr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/mock_crs_transformer.py b/tests_v2/core/data/mock_crs_transformer.py new file mode 100644 index 000000000..d9d1e40ba --- /dev/null +++ b/tests_v2/core/data/mock_crs_transformer.py @@ -0,0 +1,14 @@ +from rastervision2.core.data import CRSTransformer + + +class DoubleCRSTransformer(CRSTransformer): + """Mock CRSTransformer used for testing. + + Assumes map coords are 2x pixels coords. + """ + + def map_to_pixel(self, web_point): + return (web_point[0] * 2.0, web_point[1] * 2.0) + + def pixel_to_map(self, pixel_point): + return (pixel_point[0] / 2.0, pixel_point[1] / 2.0) diff --git a/tests_v2/core/data/mock_raster_source.py b/tests_v2/core/data/mock_raster_source.py new file mode 100644 index 000000000..601659ac1 --- /dev/null +++ b/tests_v2/core/data/mock_raster_source.py @@ -0,0 +1,51 @@ +from unittest.mock import Mock +import numpy as np + +from rastervision2.core import Box +from rastervision2.core.data import (RasterSource, IdentityCRSTransformer) +from rastervision.data import (ActivateMixin) + + +class MockRasterSource(RasterSource, ActivateMixin): + def __init__(self, channel_order, num_channels, raster_transformers=[]): + super().__init__(channel_order, num_channels, raster_transformers) + self.mock = Mock() + self.set_return_vals() + + def set_return_vals(self, raster=None): + self.mock.get_extent.return_value = Box.make_square(0, 0, 2) + self.mock.get_dtype.return_value = np.uint8 + self.mock.get_crs_transformer.return_value = IdentityCRSTransformer() + self.mock._get_chip.return_value = np.random.rand(1, 2, 2, 3) + + if raster is not None: + self.mock.get_extent.return_value = Box(0, 0, raster.shape[0], + raster.shape[1]) + self.mock.get_dtype.return_value = raster.dtype + + def get_chip(window): + return raster[window.ymin:window.ymax, window.xmin: + window.xmax, :] + + self.mock._get_chip.side_effect = get_chip + + def get_extent(self): + return self.mock.get_extent() + + def get_dtype(self): + return self.mock.get_dtype() + + def get_crs_transformer(self): + return self.mock.get_crs_transformer() + + def _get_chip(self, window): + return self.mock._get_chip(window) + + def set_raster(self, raster): + self.set_return_vals(raster=raster) + + def _activate(self): + pass + + def _deactivate(self): + pass diff --git a/tests_v2/core/data/raster_source/__init__.py b/tests_v2/core/data/raster_source/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/data/raster_source/test_rasterio_source.py b/tests_v2/core/data/raster_source/test_rasterio_source.py new file mode 100644 index 000000000..abdfc9c9e --- /dev/null +++ b/tests_v2/core/data/raster_source/test_rasterio_source.py @@ -0,0 +1,254 @@ +import unittest +from os.path import join + +import numpy as np +import rasterio +from rasterio.enums import ColorInterp + +from rastervision2.core import (RasterStats) +from rastervision2.core.utils.misc import save_img +from rastervision2.core.data import ( + ChannelOrderError, RasterioSourceConfig, StatsTransformerConfig) +from rastervision2.pipeline import rv_config + +from tests_v2 import data_file_path + + +class TestRasterioSource(unittest.TestCase): + def setUp(self): + self.tmp_dir_obj = rv_config.get_tmp_dir() + self.tmp_dir = self.tmp_dir_obj.name + + def tearDown(self): + self.tmp_dir_obj.cleanup() + + def test_nodata_val(self): + # make geotiff filled with ones and zeros with nodata == 1 + img_path = join(self.tmp_dir, 'tmp.tif') + height = 100 + width = 100 + nb_channels = 3 + with rasterio.open( + img_path, + 'w', + driver='GTiff', + height=height, + width=width, + count=nb_channels, + dtype=np.uint8, + nodata=1) as img_dataset: + im = np.random.randint( + 0, 2, (height, width, nb_channels)).astype(np.uint8) + for channel in range(nb_channels): + img_dataset.write(im[:, :, channel], channel + 1) + + config = RasterioSourceConfig(uris=[img_path]) + source = config.build(tmp_dir=self.tmp_dir) + with source.activate(): + out_chip = source.get_image_array() + expected_out_chip = np.zeros((height, width, nb_channels)) + np.testing.assert_equal(out_chip, expected_out_chip) + + def test_mask(self): + # make geotiff filled with ones and zeros and mask the whole image + img_path = join(self.tmp_dir, 'tmp.tif') + height = 100 + width = 100 + nb_channels = 3 + with rasterio.open( + img_path, + 'w', + driver='GTiff', + height=height, + width=width, + count=nb_channels, + dtype=np.uint8) as img_dataset: + im = np.random.randint( + 0, 2, (height, width, nb_channels)).astype(np.uint8) + for channel in range(nb_channels): + img_dataset.write(im[:, :, channel], channel + 1) + img_dataset.write_mask( + np.zeros(im.shape[0:2]).astype(np.bool)) + + config = RasterioSourceConfig(uris=[img_path]) + source = config.build(tmp_dir=self.tmp_dir) + with source.activate(): + out_chip = source.get_image_array() + expected_out_chip = np.zeros((height, width, nb_channels)) + np.testing.assert_equal(out_chip, expected_out_chip) + + def test_get_dtype(self): + img_path = data_file_path('small-rgb-tile.tif') + config = RasterioSourceConfig(uris=[img_path]) + source = config.build(tmp_dir=self.tmp_dir) + self.assertEqual(source.get_dtype(), np.uint8) + + def test_gets_raw_chip(self): + img_path = data_file_path('small-rgb-tile.tif') + channel_order = [0, 1] + + config = RasterioSourceConfig(uris=[img_path], channel_order=channel_order) + source = config.build(tmp_dir=self.tmp_dir) + with source.activate(): + out_chip = source.get_raw_image_array() + self.assertEqual(out_chip.shape[2], 3) + + def test_shift_x(self): + # Specially-engineered image w/ one meter per pixel resolution + # in the x direction. + img_path = data_file_path('ones.tif') + channel_order = [0] + + config = RasterioSourceConfig( + uris=[img_path], channel_order=channel_order, + x_shift=1.0, y_shift=0.0) + source = config.build(tmp_dir=self.tmp_dir) + + with source.activate(): + extent = source.get_extent() + data = source.get_chip(extent) + self.assertEqual(data.sum(), 2**16 - 256) + column = data[:, 255, 0] + self.assertEqual(column.sum(), 0) + + def test_shift_y(self): + # Specially-engineered image w/ one meter per pixel resolution + # in the y direction. + img_path = data_file_path('ones.tif') + channel_order = [0] + + config = RasterioSourceConfig( + uris=[img_path], channel_order=channel_order, + x_shift=0.0, y_shift=1.0) + source = config.build(tmp_dir=self.tmp_dir) + + with source.activate(): + extent = source.get_extent() + data = source.get_chip(extent) + self.assertEqual(data.sum(), 2**16 - 256) + row = data[0, :, 0] + self.assertEqual(row.sum(), 0) + + def test_gets_raw_chip_from_uint16_transformed_proto(self): + img_path = data_file_path('small-uint16-tile.tif') + channel_order = [0, 1] + + config = RasterioSourceConfig(uris=[img_path]) + raw_rs = config.build(tmp_dir=self.tmp_dir) + + stats_uri = join(self.tmp_dir, 'tmp.tif') + stats = RasterStats() + stats.compute([raw_rs]) + stats.save(stats_uri) + + transformer = StatsTransformerConfig(stats_uri=stats_uri) + config = RasterioSourceConfig( + uris=[img_path], + channel_order=channel_order, + transformers=[transformer]) + rs = config.build(tmp_dir=self.tmp_dir) + + with rs.activate(): + out_chip = rs.get_raw_image_array() + self.assertEqual(out_chip.shape[2], 3) + + def test_uses_channel_order(self): + img_path = join(self.tmp_dir, 'img.tif') + chip = np.ones((2, 2, 4)).astype(np.uint8) + chip[:, :, :] *= np.array([0, 1, 2, 3]).astype(np.uint8) + save_img(chip, img_path) + + channel_order = [0, 1, 2] + config = RasterioSourceConfig( + uris=[img_path], channel_order=channel_order) + source = config.build(tmp_dir=self.tmp_dir) + + with source.activate(): + out_chip = source.get_image_array() + expected_out_chip = np.ones((2, 2, 3)).astype(np.uint8) + expected_out_chip[:, :, :] *= np.array([0, 1, + 2]).astype(np.uint8) + np.testing.assert_equal(out_chip, expected_out_chip) + + def test_channel_order_error(self): + img_path = join(self.tmp_dir, 'img.tif') + chip = np.ones((2, 2, 3)).astype(np.uint8) + chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8) + save_img(chip, img_path) + + channel_order = [3, 1, 0] + with self.assertRaises(ChannelOrderError): + config = RasterioSourceConfig( + uris=[img_path], channel_order=channel_order) + config.build(tmp_dir=self.tmp_dir) + + def test_detects_alpha(self): + # Set first channel to alpha. Expectation is that when omitting channel_order, + # only the second and third channels will be in output. + img_path = join(self.tmp_dir, 'img.tif') + chip = np.ones((2, 2, 3)).astype(np.uint8) + chip[:, :, :] *= np.array([0, 1, 2]).astype(np.uint8) + save_img(chip, img_path) + + ci = (ColorInterp.alpha, ColorInterp.blue, ColorInterp.green) + with rasterio.open(img_path, 'r+') as src: + src.colorinterp = ci + + config = RasterioSourceConfig(uris=[img_path]) + source = config.build(tmp_dir=self.tmp_dir) + with source.activate(): + out_chip = source.get_image_array() + expected_out_chip = np.ones((2, 2, 2)).astype(np.uint8) + expected_out_chip[:, :, :] *= np.array([1, 2]).astype(np.uint8) + np.testing.assert_equal(out_chip, expected_out_chip) + + def test_non_geo(self): + # Check if non-georeferenced image files can be read and CRSTransformer + # implements the identity function. + img_path = join(self.tmp_dir, 'img.png') + chip = np.ones((2, 2, 3)).astype(np.uint8) + save_img(chip, img_path) + + config = RasterioSourceConfig(uris=[img_path]) + source = config.build(tmp_dir=self.tmp_dir) + with source.activate(): + out_chip = source.get_image_array() + np.testing.assert_equal(out_chip, chip) + + p = (3, 4) + out_p = source.get_crs_transformer().map_to_pixel(p) + np.testing.assert_equal(out_p, p) + + out_p = source.get_crs_transformer().pixel_to_map(p) + np.testing.assert_equal(out_p, p) + + def test_no_epsg(self): + crs = rasterio.crs.CRS() + img_path = join(self.tmp_dir, 'tmp.tif') + height = 100 + width = 100 + nb_channels = 3 + with rasterio.open( + img_path, + 'w', + driver='GTiff', + height=height, + width=width, + count=nb_channels, + dtype=np.uint8, + crs=crs) as img_dataset: + im = np.zeros((height, width, nb_channels)).astype(np.uint8) + for channel in range(nb_channels): + img_dataset.write(im[:, :, channel], channel + 1) + + try: + config = RasterioSourceConfig(uris=[img_path]) + config.build(tmp_dir=self.tmp_dir) + except Exception: + self.fail( + 'Creating RasterioSource with CRS with no EPSG attribute ' + 'raised an exception when it should not have.') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/raster_source/test_rasterized_source.py b/tests_v2/core/data/raster_source/test_rasterized_source.py new file mode 100644 index 000000000..05eec1aae --- /dev/null +++ b/tests_v2/core/data/raster_source/test_rasterized_source.py @@ -0,0 +1,140 @@ +import unittest +from os.path import join + +import numpy as np + +from rastervision2.core import Box +from rastervision2.core.data import ( + IdentityCRSTransformer, RasterizedSourceConfig, RasterizerConfig, + GeoJSONVectorSourceConfig, ClassConfig) +from rastervision2.pipeline.file_system import json_to_file +from rastervision2.pipeline.config import ConfigError +from rastervision2.pipeline import rv_config + + +class TestRasterizedSource(unittest.TestCase): + def setUp(self): + self.crs_transformer = IdentityCRSTransformer() + self.extent = Box.make_square(0, 0, 10) + self.tmp_dir_obj = rv_config.get_tmp_dir() + self.tmp_dir = self.tmp_dir_obj.name + self.class_id = 0 + self.background_class_id = 1 + self.line_buffer = 1 + self.class_config = ClassConfig(names=['a']) + self.uri = join(self.tmp_dir, 'tmp.json') + + def tearDown(self): + self.tmp_dir_obj.cleanup() + + def build_source(self, geojson, all_touched=False): + json_to_file(geojson, self.uri) + + config = RasterizedSourceConfig( + vector_source=GeoJSONVectorSourceConfig(uri=self.uri, default_class_id=None), + rasterizer_config=RasterizerConfig( + background_class_id=self.background_class_id, + all_touched=all_touched)) + source = config.build(self.class_config, self.crs_transformer, self.extent) + return source + + def test_get_chip(self): + geojson = { + 'type': + 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'geometry': { + 'type': + 'Polygon', + 'coordinates': [[[0., 0.], [0., 5.], [5., 5.], [5., 0.], + [0., 0.]]] + }, + 'properties': { + 'class_id': self.class_id, + } + }, { + 'type': 'Feature', + 'geometry': { + 'type': 'LineString', + 'coordinates': [[7., 0.], [7., 9.]] + }, + 'properties': { + 'class_id': self.class_id + } + }] + } + + source = self.build_source(geojson) + with source.activate(): + self.assertEqual(source.get_extent(), self.extent) + chip = source.get_image_array() + self.assertEqual(chip.shape, (10, 10, 1)) + + expected_chip = self.background_class_id * np.ones((10, 10, 1)) + expected_chip[0:5, 0:5, 0] = self.class_id + expected_chip[0:10, 6:8] = self.class_id + np.testing.assert_array_equal(chip, expected_chip) + + def test_get_chip_no_polygons(self): + geojson = {'type': 'FeatureCollection', 'features': []} + + source = self.build_source(geojson) + with source.activate(): + # Get chip that partially overlaps extent. Expect that chip has zeros + # outside of extent, and background_class_id otherwise. + self.assertEqual(source.get_extent(), self.extent) + chip = source.get_chip(Box.make_square(5, 5, 10)) + self.assertEqual(chip.shape, (10, 10, 1)) + + expected_chip = np.zeros((10, 10, 1)) + expected_chip[0:5, 0:5, :] = self.background_class_id + + np.testing.assert_array_equal(chip, expected_chip) + + def test_get_chip_all_touched(self): + geojson = { + 'type': + 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'geometry': { + 'type': + 'Polygon', + 'coordinates': [[[0., 0.], [0., 0.4], [0.4, 0.4], + [0.4, 0.], [0., 0.]]] + }, + 'properties': { + 'class_id': self.class_id, + } + }] + } + + false_source = self.build_source(geojson, all_touched=False) + true_source = self.build_source(geojson, all_touched=True) + with false_source.activate(): + chip = false_source.get_image_array() + expected_chip = self.background_class_id * np.ones((10, 10, 1)) + np.testing.assert_array_equal(chip, expected_chip) + + with true_source.activate(): + chip = true_source.get_image_array() + expected_chip = self.background_class_id * np.ones((10, 10, 1)) + expected_chip[0:1, 0:1, 0] = self.class_id + np.testing.assert_array_equal(chip, expected_chip) + + def test_using_null_class_bufs(self): + vs = GeoJSONVectorSourceConfig( + uri=self.uri, + default_class_id=None, + line_bufs={0: None}) + with self.assertRaises(ConfigError): + config = RasterizedSourceConfig( + vector_source=vs, + rasterizer_config=RasterizerConfig( + background_class_id=self.background_class_id)) + config.validate_config() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/raster_transformer/__init__.py b/tests_v2/core/data/raster_transformer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/data/raster_transformer/test_raster_transformer.py b/tests_v2/core/data/raster_transformer/test_raster_transformer.py new file mode 100644 index 000000000..6b519e0a9 --- /dev/null +++ b/tests_v2/core/data/raster_transformer/test_raster_transformer.py @@ -0,0 +1,30 @@ +import unittest +import os + +import numpy as np + +from rastervision2.core.data import RasterStats, StatsTransformerConfig +from rastervision2.pipeline import rv_config + + +class TestRasterTransformer(unittest.TestCase): + def test_stats_transformer(self): + raster_stats = RasterStats() + raster_stats.means = list(np.ones((4, ))) + raster_stats.stds = list(np.ones((4, )) * 2) + + with rv_config.get_tmp_dir() as tmp_dir: + stats_uri = os.path.join(tmp_dir, 'stats.json') + raster_stats.save(stats_uri) + + # All values have z-score of 1, which translates to + # uint8 value of 170. + transformer = StatsTransformerConfig(stats_uri=stats_uri).build() + chip = np.ones((2, 2, 4)) * 3 + out_chip = transformer.transform(chip) + expected_out_chip = np.ones((2, 2, 4)) * 170 + np.testing.assert_equal(out_chip, expected_out_chip) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/data/vector_source/__init__.py b/tests_v2/core/data/vector_source/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/data/vector_source/test_geojson_vector_source.py b/tests_v2/core/data/vector_source/test_geojson_vector_source.py new file mode 100644 index 000000000..97acb82ee --- /dev/null +++ b/tests_v2/core/data/vector_source/test_geojson_vector_source.py @@ -0,0 +1,185 @@ +import unittest +import os + +from shapely.geometry import shape + +from rastervision2.core.data import ( + GeoJSONVectorSourceConfig, ClassConfig, IdentityCRSTransformer) +from rastervision2.pipeline.file_system import json_to_file +from rastervision2.pipeline import rv_config + +from tests_v2.core.data.mock_crs_transformer import DoubleCRSTransformer + + +class TestGeoJSONVectorSource(unittest.TestCase): + """This also indirectly tests the ClassInference class.""" + + def setUp(self): + self.tmp_dir = rv_config.get_tmp_dir() + self.uri = os.path.join(self.tmp_dir.name, 'vectors.json') + + def tearDown(self): + self.tmp_dir.cleanup() + + def _test_class_inf(self, props, exp_class_ids, default_class_id=None): + geojson = { + 'type': + 'FeatureCollection', + 'features': [{ + 'properties': props, + 'geometry': { + 'type': 'Point', + 'coordinates': [1, 1] + } + }] + } + json_to_file(geojson, self.uri) + + class_config = ClassConfig(names=['building', 'car', 'tree']) + class_id_to_filter = { + 0: ['==', 'type', 'building'], + 1: ['any', ['==', 'type', 'car'], ['==', 'type', 'auto']] + } + vs_cfg = GeoJSONVectorSourceConfig( + uri=self.uri, + class_id_to_filter=class_id_to_filter, + default_class_id=default_class_id) + vs = vs_cfg.build(class_config, IdentityCRSTransformer()) + trans_geojson = vs.get_geojson() + class_ids = [ + f['properties']['class_id'] for f in trans_geojson['features'] + ] + self.assertEqual(class_ids, exp_class_ids) + + def test_class_inf_class_id(self): + self._test_class_inf({'class_id': 2}, [2]) + + def test_class_inf_label(self): + self._test_class_inf({'label': 'car'}, [1]) + + def test_class_inf_filter(self): + self._test_class_inf({'type': 'auto'}, [1]) + + def test_class_inf_default(self): + self._test_class_inf({}, [3], default_class_id=3) + + def test_class_inf_no_default(self): + self._test_class_inf({}, []) + + def geom_to_geojson(self, geom): + return {'type': 'FeatureCollection', 'features': [{'geometry': geom}]} + + def transform_geojson(self, + geojson, + line_bufs=None, + point_bufs=None, + crs_transformer=None, + to_map_coords=False): + if crs_transformer is None: + crs_transformer = IdentityCRSTransformer() + class_config = ClassConfig(names=['building']) + json_to_file(geojson, self.uri) + cfg = GeoJSONVectorSourceConfig( + uri=self.uri, + line_bufs=line_bufs, + point_bufs=point_bufs, + default_class_id=0) + source = cfg.build(class_config, crs_transformer) + return source.get_geojson(to_map_coords=to_map_coords) + + def test_transform_geojson_no_coords(self): + geom = {'type': 'Point', 'coordinates': []} + geojson = self.geom_to_geojson(geom) + trans_geojson = self.transform_geojson(geojson) + + self.assertEqual(0, len(trans_geojson['features'])) + + def test_transform_geojson_geom_coll(self): + geom = { + 'type': + 'GeometryCollection', + 'geometries': [{ + 'type': 'MultiPoint', + 'coordinates': [[10, 10], [20, 20]] + }] + } + geojson = self.geom_to_geojson(geom) + trans_geojson = self.transform_geojson(geojson) + + feats = trans_geojson['features'] + self.assertEqual(len(feats), 2) + self.assertEqual(feats[0]['geometry']['type'], 'Polygon') + self.assertEqual(feats[1]['geometry']['type'], 'Polygon') + + def test_transform_geojson_multi(self): + geom = {'type': 'MultiPoint', 'coordinates': [[10, 10], [20, 20]]} + geojson = self.geom_to_geojson(geom) + trans_geojson = self.transform_geojson(geojson) + + feats = trans_geojson['features'] + self.assertEqual(len(feats), 2) + self.assertEqual(feats[0]['geometry']['type'], 'Polygon') + self.assertEqual(feats[1]['geometry']['type'], 'Polygon') + + def test_transform_geojson_line_buf(self): + geom = {'type': 'LineString', 'coordinates': [[10, 10], [10, 20]]} + geojson = self.geom_to_geojson(geom) + + trans_geojson = self.transform_geojson(geojson, line_bufs={0: 5.0}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).buffer(5.0).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson(geojson, line_bufs={1: 5.0}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).buffer(1.0).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson(geojson, line_bufs={0: None}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).equals(shape(trans_geom))) + + def test_transform_point_buf(self): + geom = {'type': 'Point', 'coordinates': [10, 10]} + geojson = self.geom_to_geojson(geom) + + trans_geojson = self.transform_geojson(geojson, point_bufs={0: 5.0}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).buffer(5.0).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson(geojson, point_bufs={1: 5.0}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).buffer(1.0).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson(geojson, point_bufs={0: None}) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).equals(shape(trans_geom))) + + def test_transform_polygon(self): + geom = { + 'type': 'Polygon', + 'coordinates': [[[0, 0], [0, 10], [10, 10], [10, 0], [0, 0]]] + } + geojson = self.geom_to_geojson(geom) + + trans_geojson = self.transform_geojson(geojson) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson( + geojson, crs_transformer=DoubleCRSTransformer()) + trans_geom = trans_geojson['features'][0]['geometry'] + exp_geom = { + 'type': 'Polygon', + 'coordinates': [[[0, 0], [0, 20], [20, 20], [20, 0], [0, 0]]] + } + self.assertTrue(shape(exp_geom).equals(shape(trans_geom))) + + trans_geojson = self.transform_geojson( + geojson, + crs_transformer=DoubleCRSTransformer(), + to_map_coords=True) + trans_geom = trans_geojson['features'][0]['geometry'] + self.assertTrue(shape(geom).equals(shape(trans_geom))) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/__init__.py b/tests_v2/core/evaluation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/core/evaluation/test_chip_classification_evaluation.py b/tests_v2/core/evaluation/test_chip_classification_evaluation.py new file mode 100644 index 000000000..88b8d9a5f --- /dev/null +++ b/tests_v2/core/evaluation/test_chip_classification_evaluation.py @@ -0,0 +1,79 @@ +import unittest + +from rastervision2.core.evaluation import ChipClassificationEvaluation +from rastervision2.core.data.class_config import ClassConfig +from rastervision2.core import Box +from rastervision2.core.data.label import ChipClassificationLabels + + +class TestChipClassificationEvaluation(unittest.TestCase): + def make_class_config(self): + return ClassConfig(names=['grassy', 'urban']) + + def make_labels(self, class_ids): + """Make 2x2 grid label store. + + Args: + class_ids: 2x2 array of class_ids to use + """ + cell_size = 200 + y_cells = 2 + x_cells = 2 + labels = ChipClassificationLabels() + + for yind in range(y_cells): + for xind in range(x_cells): + ymin = yind * cell_size + xmin = xind * cell_size + ymax = ymin + cell_size + xmax = xmin + cell_size + window = Box(ymin, xmin, ymax, xmax) + class_id = class_ids[yind][xind] + new_labels = ChipClassificationLabels() + new_labels.set_cell(window, class_id) + labels.extend(new_labels) + + return labels + + def assert_eval_single_null(self, eval): + eval_item0 = eval.class_to_eval_item[0] + self.assertEqual(eval_item0.gt_count, 2) + self.assertEqual(eval_item0.precision, 1.0) + self.assertEqual(eval_item0.recall, 0.5) + self.assertAlmostEqual(eval_item0.f1, 2 / 3, places=2) + + eval_item1 = eval.class_to_eval_item[1] + self.assertEqual(eval_item1.gt_count, 1) + self.assertEqual(eval_item1.precision, 0.5) + self.assertEqual(eval_item1.recall, 1.0) + self.assertAlmostEqual(eval_item1.f1, 2 / 3, places=2) + + avg_item = eval.avg_item + self.assertEqual(avg_item.gt_count, 3) + self.assertAlmostEqual(avg_item.precision, 0.83, places=2) + self.assertAlmostEqual(avg_item.recall, 2 / 3, places=2) + self.assertAlmostEqual(avg_item.f1, 2 / 3, places=2) + + def test_compute_single_pred_null(self): + class_config = self.make_class_config() + eval = ChipClassificationEvaluation(class_config) + gt_class_ids = [[0, 1], [0, 1]] + gt_labels = self.make_labels(gt_class_ids) + pred_class_ids = [[0, None], [1, 1]] + pred_labels = self.make_labels(pred_class_ids) + eval.compute(gt_labels, pred_labels) + self.assert_eval_single_null(eval) + + def test_compute_single_gt_null(self): + class_config = self.make_class_config() + eval = ChipClassificationEvaluation(class_config) + gt_class_ids = [[0, None], [0, 1]] + gt_labels = self.make_labels(gt_class_ids) + pred_class_ids = [[0, 1], [1, 1]] + pred_labels = self.make_labels(pred_class_ids) + eval.compute(gt_labels, pred_labels) + self.assert_eval_single_null(eval) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/test_chip_classification_evaluator.py b/tests_v2/core/evaluation/test_chip_classification_evaluator.py new file mode 100644 index 000000000..0b1d4e3e9 --- /dev/null +++ b/tests_v2/core/evaluation/test_chip_classification_evaluator.py @@ -0,0 +1,51 @@ +import unittest +import os + +from rastervision2.pipeline import rv_config +from rastervision2.pipeline.file_system import file_to_json +from rastervision2.core.data import ( + ClassConfig, ChipClassificationLabelSourceConfig, GeoJSONVectorSourceConfig, + ChipClassificationGeoJSONStoreConfig, RasterioSourceConfig, SceneConfig) +from rastervision2.core.evaluation import (ChipClassificationEvaluatorConfig) + +from tests_v2 import data_file_path + + +class TestChipClassificationEvaluator(unittest.TestCase): + def test_accounts_for_aoi(self): + class_config = ClassConfig(names=['car', 'building', 'background']) + + label_source_uri = data_file_path('evaluator/cc-label-filtered.json') + label_source_cfg = ChipClassificationLabelSourceConfig( + vector_source=GeoJSONVectorSourceConfig( + uri=label_source_uri, default_class_id=None)) + + label_store_uri = data_file_path('evaluator/cc-label-full.json') + label_store_cfg = ChipClassificationGeoJSONStoreConfig(uri=label_store_uri) + + raster_source_uri = data_file_path('evaluator/cc-label-img-blank.tif') + raster_source_cfg = RasterioSourceConfig(uris=[raster_source_uri]) + + aoi_uri = data_file_path('evaluator/cc-label-aoi.json') + s = SceneConfig( + id='test', + raster_source=raster_source_cfg, + label_source=label_source_cfg, + label_store=label_store_cfg, + aoi_uris=[aoi_uri]) + + with rv_config.get_tmp_dir() as tmp_dir: + scene = s.build(class_config, tmp_dir) + output_uri = os.path.join(tmp_dir, 'eval.json') + + evaluator = ChipClassificationEvaluatorConfig( + output_uri=output_uri).build(class_config) + evaluator.process([scene], tmp_dir) + + overall = file_to_json(output_uri)['overall'] + for item in overall: + self.assertEqual(item['f1'], 1.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/test_class_evaluation_item.py b/tests_v2/core/evaluation/test_class_evaluation_item.py new file mode 100644 index 000000000..fe57fae77 --- /dev/null +++ b/tests_v2/core/evaluation/test_class_evaluation_item.py @@ -0,0 +1,56 @@ +import unittest + +from rastervision2.core.evaluation import ClassEvaluationItem + + +class TestClassEvaluationItem(unittest.TestCase): + def setUp(self): + pass + + def test_merge_both_empty(self): + a = ClassEvaluationItem() + b = ClassEvaluationItem() + a.merge(b) + self.assertEqual(a.precision, None) + self.assertEqual(a.recall, None) + self.assertEqual(a.f1, None) + self.assertEqual(a.count_error, None) + self.assertEqual(a.gt_count, 0) + + def test_merge_first_empty(self): + a = ClassEvaluationItem() + b = ClassEvaluationItem( + precision=1, recall=1, f1=1, count_error=0, gt_count=1) + a.merge(b) + self.assertEqual(a.precision, 1) + self.assertEqual(a.recall, 1) + self.assertEqual(a.f1, 1) + self.assertEqual(a.count_error, 0) + self.assertEqual(a.gt_count, 1) + + def test_merge_second_empty(self): + a = ClassEvaluationItem( + precision=1, recall=1, f1=1, count_error=0, gt_count=1) + b = ClassEvaluationItem() + a.merge(b) + self.assertEqual(a.precision, 1) + self.assertEqual(a.recall, 1) + self.assertEqual(a.f1, 1) + self.assertEqual(a.count_error, 0) + self.assertEqual(a.gt_count, 1) + + def test_merge(self): + a = ClassEvaluationItem( + precision=1, recall=1, f1=1, count_error=0, gt_count=1) + b = ClassEvaluationItem( + precision=0, recall=0, f1=0, count_error=1, gt_count=2) + a.merge(b) + self.assertEqual(a.precision, 1 / 3) + self.assertEqual(a.recall, 1 / 3) + self.assertEqual(a.f1, 1 / 3) + self.assertEqual(a.count_error, 2 / 3) + self.assertEqual(a.gt_count, 3) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/test_object_detection_evaluation.py b/tests_v2/core/evaluation/test_object_detection_evaluation.py new file mode 100644 index 000000000..098335fdb --- /dev/null +++ b/tests_v2/core/evaluation/test_object_detection_evaluation.py @@ -0,0 +1,113 @@ +import unittest + +import numpy as np + +from rastervision2.core.evaluation import ObjectDetectionEvaluation +from rastervision2.core.data import ClassConfig, ObjectDetectionLabels +from rastervision2.core import Box + + +class TestObjectDetectionEvaluation(unittest.TestCase): + def make_class_config(self): + return ClassConfig(names=['name', 'building']) + + def make_ground_truth_labels(self): + size = 100 + nw = Box.make_square(0, 0, size) + ne = Box.make_square(0, 200, size) + se = Box.make_square(200, 200, size) + sw = Box.make_square(200, 0, size) + npboxes = Box.to_npboxes([nw, ne, se, sw]) + class_ids = np.array([0, 0, 1, 1]) + return ObjectDetectionLabels(npboxes, class_ids) + + def make_predicted_labels(self): + size = 100 + # Predicted labels are only there for three of the ground truth boxes, + # and are offset by 10 pixels. + nw = Box.make_square(10, 0, size) + ne = Box.make_square(10, 200, size) + se = Box.make_square(210, 200, size) + npboxes = Box.to_npboxes([nw, ne, se]) + class_ids = np.array([0, 0, 1]) + scores = np.ones(class_ids.shape) + return ObjectDetectionLabels(npboxes, class_ids, scores=scores) + + def test_compute(self): + class_config = self.make_class_config() + eval = ObjectDetectionEvaluation(class_config) + gt_labels = self.make_ground_truth_labels() + pred_labels = self.make_predicted_labels() + + eval.compute(gt_labels, pred_labels) + eval_item1 = eval.class_to_eval_item[0] + self.assertEqual(eval_item1.gt_count, 2) + self.assertEqual(eval_item1.precision, 1.0) + self.assertEqual(eval_item1.recall, 1.0) + self.assertEqual(eval_item1.f1, 1.0) + + eval_item2 = eval.class_to_eval_item[1] + self.assertEqual(eval_item2.gt_count, 2) + self.assertEqual(eval_item2.precision, 1.0) + self.assertEqual(eval_item2.recall, 0.5) + self.assertEqual(eval_item2.f1, 2 / 3) + + avg_item = eval.avg_item + self.assertEqual(avg_item.gt_count, 4) + self.assertAlmostEqual(avg_item.precision, 1.0) + self.assertEqual(avg_item.recall, 0.75) + self.assertAlmostEqual(avg_item.f1, 0.83, places=2) + + def test_compute_no_preds(self): + class_config = self.make_class_config() + eval = ObjectDetectionEvaluation(class_config) + gt_labels = self.make_ground_truth_labels() + pred_labels = ObjectDetectionLabels.make_empty() + + eval.compute(gt_labels, pred_labels) + eval_item1 = eval.class_to_eval_item[0] + self.assertEqual(eval_item1.gt_count, 2) + self.assertEqual(eval_item1.precision, None) + self.assertEqual(eval_item1.recall, 0.0) + self.assertEqual(eval_item1.f1, None) + + eval_item2 = eval.class_to_eval_item[1] + self.assertEqual(eval_item2.gt_count, 2) + self.assertEqual(eval_item2.precision, None) + self.assertEqual(eval_item2.recall, 0.0) + self.assertEqual(eval_item2.f1, None) + + avg_item = eval.avg_item + self.assertEqual(avg_item.gt_count, 4) + self.assertEqual(avg_item.precision, 0.0) + self.assertEqual(avg_item.recall, 0.0) + self.assertEqual(avg_item.f1, 0.0) + + def test_compute_no_ground_truth(self): + class_config = self.make_class_config() + eval = ObjectDetectionEvaluation(class_config) + gt_labels = ObjectDetectionLabels.make_empty() + pred_labels = self.make_predicted_labels() + + eval.compute(gt_labels, pred_labels) + eval_item1 = eval.class_to_eval_item[0] + self.assertEqual(eval_item1.gt_count, 0) + self.assertEqual(eval_item1.precision, None) + self.assertEqual(eval_item1.recall, None) + self.assertEqual(eval_item1.f1, None) + + eval_item2 = eval.class_to_eval_item[1] + self.assertEqual(eval_item2.gt_count, 0) + self.assertEqual(eval_item2.precision, None) + self.assertEqual(eval_item2.recall, None) + self.assertEqual(eval_item2.f1, None) + + avg_item = eval.avg_item + self.assertEqual(avg_item.gt_count, 0) + self.assertEqual(avg_item.precision, None) + self.assertEqual(avg_item.recall, None) + self.assertEqual(avg_item.f1, None) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/test_semantic_segmentation_evaluation.py b/tests_v2/core/evaluation/test_semantic_segmentation_evaluation.py new file mode 100644 index 000000000..11923881b --- /dev/null +++ b/tests_v2/core/evaluation/test_semantic_segmentation_evaluation.py @@ -0,0 +1,109 @@ +import unittest + +import numpy as np + +from rastervision2.core.data import ClassConfig +from rastervision2.core.evaluation import SemanticSegmentationEvaluation +from rastervision2.core.data import SemanticSegmentationLabelSource +from tests_v2.core.data.mock_raster_source import MockRasterSource +from tests_v2 import data_file_path + + +class TestSemanticSegmentationEvaluation(unittest.TestCase): + def test_compute(self): + class_config = ClassConfig(names=['one', 'two']) + class_config.update() + class_config.ensure_null_class() + + gt_array = np.zeros((4, 4, 1), dtype=np.uint8) + gt_array[2, 2, 0] = 1 + gt_array[0, 0, 0] = 2 + gt_raster = MockRasterSource([0], 1) + gt_raster.set_raster(gt_array) + gt_label_source = SemanticSegmentationLabelSource(raster_source=gt_raster) + + p_array = np.zeros((4, 4, 1), dtype=np.uint8) + p_array[1, 1, 0] = 1 + p_raster = MockRasterSource([0], 1) + p_raster.set_raster(p_array) + p_label_source = SemanticSegmentationLabelSource(raster_source=p_raster) + + eval = SemanticSegmentationEvaluation(class_config) + eval.compute(gt_label_source.get_labels(), p_label_source.get_labels()) + + tp0 = 16 - 3 # 4*4 - 3 true positives for class 0 + fp0 = 1 # 1 false positive (2,2) and one don't care at (0,0) + fn0 = 1 # one false negative (1,1) + precision0 = float(tp0) / (tp0 + fp0) + recall0 = float(tp0) / (tp0 + fn0) + f10 = 2 * float(precision0 * recall0) / (precision0 + recall0) + + tp1 = 0 # 0 true positives for class 1 + fn1 = 1 # one false negative (2,2) + precision1 = 0 # float(tp1) / (tp1 + fp1) where fp1 == 1 + recall1 = float(tp1) / (tp1 + fn1) + f11 = None + + self.assertAlmostEqual(precision0, + eval.class_to_eval_item[0].precision) + self.assertAlmostEqual(recall0, eval.class_to_eval_item[0].recall) + self.assertAlmostEqual(f10, eval.class_to_eval_item[0].f1) + + self.assertEqual(precision1, eval.class_to_eval_item[1].precision) + self.assertAlmostEqual(recall1, eval.class_to_eval_item[1].recall) + self.assertAlmostEqual(f11, eval.class_to_eval_item[1].f1) + + avg_conf_mat = np.array([[0, 0, 0], [13., 1, 0], [1, 0, 0]]) + avg_recall = (14 / 15) * recall0 + (1 / 15) * recall1 + self.assertTrue(np.array_equal(avg_conf_mat, eval.avg_item.conf_mat)) + self.assertEqual(avg_recall, eval.avg_item.recall) + + def test_compute_ignore_class(self): + class_config = ClassConfig(names=['one', 'two']) + class_config.update() + class_config.ensure_null_class() + + gt_array = np.zeros((4, 4, 1), dtype=np.uint8) + gt_array[0, 0, 0] = 2 + gt_raster = MockRasterSource([0], 1) + gt_raster.set_raster(gt_array) + gt_label_source = SemanticSegmentationLabelSource(raster_source=gt_raster) + + pred_array = np.zeros((4, 4, 1), dtype=np.uint8) + pred_array[0, 0, 0] = 1 + pred_raster = MockRasterSource([0], 1) + pred_raster.set_raster(pred_array) + pred_label_source = SemanticSegmentationLabelSource(raster_source=pred_raster) + + eval = SemanticSegmentationEvaluation(class_config) + eval.compute(gt_label_source.get_labels(), + pred_label_source.get_labels()) + self.assertAlmostEqual(1.0, eval.class_to_eval_item[0].f1) + self.assertAlmostEqual(1.0, eval.avg_item.f1) + + def test_vector_compute(self): + class_config = ClassConfig(names=['one', 'two']) + class_config.update() + class_config.ensure_null_class() + + gt_uri = data_file_path('2-gt-polygons.geojson') + pred_uri = data_file_path('2-pred-polygons.geojson') + + eval = SemanticSegmentationEvaluation(class_config) + eval.compute_vector(gt_uri, pred_uri, 'polygons', 0) + + # NOTE: The two geojson files referenced above contain three + # unique geometries total, each file contains two geometries, + # and there is one geometry shared between the two. + tp = 1.0 + fp = 1.0 + fn = 1.0 + precision = float(tp) / (tp + fp) + recall = float(tp) / (tp + fn) + + self.assertAlmostEqual(precision, eval.class_to_eval_item[0].precision) + self.assertAlmostEqual(recall, eval.class_to_eval_item[0].recall) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/evaluation/test_semantic_segmentation_evaluator.py b/tests_v2/core/evaluation/test_semantic_segmentation_evaluator.py new file mode 100644 index 000000000..fe150c49f --- /dev/null +++ b/tests_v2/core/evaluation/test_semantic_segmentation_evaluator.py @@ -0,0 +1,135 @@ +import unittest +from os.path import join + +import numpy as np +from shapely.geometry import shape + +from rastervision2.core.data import ClassConfig +from rastervision2.core import Box +from rastervision2.core.data import ( + Scene, IdentityCRSTransformer, + SemanticSegmentationLabelSource, RasterizedSourceConfig, RasterizerConfig, + GeoJSONVectorSourceConfig, PolygonVectorOutputConfig) +from rastervision2.core.evaluation import SemanticSegmentationEvaluator +from rastervision2.pipeline import rv_config +from rastervision2.pipeline.file_system import file_to_json + +from tests_v2.core.data.mock_raster_source import (MockRasterSource) +from tests_v2 import data_file_path + + +class TestSemanticSegmentationEvaluator(unittest.TestCase): + def setUp(self): + self.tmp_dir = rv_config.get_tmp_dir() + + self.class_config = ClassConfig(names=['one', 'two']) + self.class_config.update() + self.class_config.ensure_null_class() + + def tearDown(self): + self.tmp_dir.cleanup() + + def get_scene(self, class_id): + # Make scene where ground truth is all set to class_id + # and predictions are set to half 0's and half 1's + scene_id = str(class_id) + rs = MockRasterSource(channel_order=[0, 1, 2], num_channels=3) + rs.set_raster(np.zeros((10, 10, 3))) + + gt_rs = MockRasterSource(channel_order=[0], num_channels=1) + gt_arr = np.full((10, 10, 1), class_id) + gt_rs.set_raster(gt_arr) + gt_ls = SemanticSegmentationLabelSource(raster_source=gt_rs) + + pred_rs = MockRasterSource(channel_order=[0], num_channels=1) + pred_arr = np.zeros((10, 10, 1)) + pred_arr[5:10, :, :] = 1 + pred_rs.set_raster(pred_arr) + pred_ls = SemanticSegmentationLabelSource(raster_source=pred_rs) + + return Scene(scene_id, rs, gt_ls, pred_ls) + + def test_evaluator(self): + output_uri = join(self.tmp_dir.name, 'out.json') + scenes = [self.get_scene(0), self.get_scene(1)] + evaluator = SemanticSegmentationEvaluator(self.class_config, output_uri, None) + evaluator.process(scenes, self.tmp_dir.name) + eval_json = file_to_json(output_uri) + exp_eval_json = file_to_json(data_file_path('expected-eval.json')) + self.assertDictEqual(eval_json, exp_eval_json) + + def get_vector_scene(self, class_id, use_aoi=False): + gt_uri = data_file_path('{}-gt-polygons.geojson'.format(class_id)) + pred_uri = data_file_path('{}-pred-polygons.geojson'.format(class_id)) + + scene_id = str(class_id) + rs = MockRasterSource(channel_order=[0, 1, 3], num_channels=3) + rs.set_raster(np.zeros((10, 10, 3))) + + crs_transformer = IdentityCRSTransformer() + extent = Box.make_square(0, 0, 360) + + config = RasterizedSourceConfig( + vector_source=GeoJSONVectorSourceConfig(uri=gt_uri, default_class_id=0), + rasterizer_config=RasterizerConfig( + background_class_id=1)) + gt_rs = config.build(self.class_config, crs_transformer, extent) + gt_ls = SemanticSegmentationLabelSource(raster_source=gt_rs) + + config = RasterizedSourceConfig( + vector_source=GeoJSONVectorSourceConfig(uri=pred_uri, default_class_id=0), + rasterizer_config=RasterizerConfig( + background_class_id=1)) + pred_rs = config.build(self.class_config, crs_transformer, extent) + pred_ls = SemanticSegmentationLabelSource(raster_source=pred_rs) + pred_ls.vector_output = [ + PolygonVectorOutputConfig( + uri=pred_uri, + denoise=0, + class_id=class_id) + ] + + if use_aoi: + aoi_uri = data_file_path('{}-aoi.geojson'.format(class_id)) + aoi_geojson = file_to_json(aoi_uri) + aoi_polygons = [shape(aoi_geojson['features'][0]['geometry'])] + return Scene(scene_id, rs, gt_ls, pred_ls, aoi_polygons) + + return Scene(scene_id, rs, gt_ls, pred_ls) + + def test_vector_evaluator(self): + output_uri = join(self.tmp_dir.name, 'raster-out.json') + vector_output_uri = join(self.tmp_dir.name, 'vector-out.json') + scenes = [self.get_vector_scene(0), self.get_vector_scene(1)] + evaluator = SemanticSegmentationEvaluator( + self.class_config, output_uri, vector_output_uri) + evaluator.process(scenes, self.tmp_dir.name) + vector_eval_json = file_to_json(vector_output_uri) + exp_vector_eval_json = file_to_json(data_file_path('expected-vector-eval.json')) + + # NOTE: The precision and recall values found in the file + # `expected-vector-eval.json` are equal to fractions of the + # form (n-1)/n for n <= 7 which can be seen to be (and have + # been manually verified to be) correct. + self.assertDictEqual(vector_eval_json, exp_vector_eval_json) + + def test_vector_evaluator_with_aoi(self): + output_uri = join(self.tmp_dir.name, 'raster-out.json') + vector_output_uri = join(self.tmp_dir.name, 'vector-out.json') + scenes = [self.get_vector_scene(0, use_aoi=True)] + evaluator = SemanticSegmentationEvaluator( + self.class_config, output_uri, vector_output_uri) + evaluator.process(scenes, self.tmp_dir.name) + vector_eval_json = file_to_json(vector_output_uri) + exp_vector_eval_json = file_to_json( + data_file_path('expected-vector-eval-with-aoi.json')) + + # NOTE: The precision and recall values found in the file + # `expected-vector-eval.json` are equal to fractions of the + # form (n-1)/n for n <= 7 which can be seen to be (and have + # been manually verified to be) correct. + self.assertDictEqual(vector_eval_json, exp_vector_eval_json) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/test_box.py b/tests_v2/core/test_box.py new file mode 100644 index 000000000..05850fec5 --- /dev/null +++ b/tests_v2/core/test_box.py @@ -0,0 +1,187 @@ +import unittest + +import numpy as np +from shapely.geometry import box as ShapelyBox + +from rastervision2.core.box import Box, BoxSizeError + +np.random.seed(1) + + +class TestBox(unittest.TestCase): + def setUp(self): + self.ymin = 0 + self.xmin = 0 + self.ymax = 2 + self.xmax = 3 + self.box = Box(self.ymin, self.xmin, self.ymax, self.xmax) + + def test_reproject(self): + def transform(point): + (y, x) = point + return ((y + 1) // 2, x // 2) + + reproj = self.box.reproject(transform) + self.assertTrue(reproj.xmin == 0) + self.assertTrue(reproj.ymin == 0) + self.assertTrue(reproj.ymax == 1) + self.assertTrue(reproj.xmax == 2) + + def test_dict(self): + dictionary = self.box.to_dict() + other = Box.from_dict(dictionary) + self.assertTrue(self.box == other) + + def test_bad_square(self): + self.assertRaises(BoxSizeError, + lambda: self.box.make_random_square(10)) + + def test_bad_conatiner(self): + self.assertRaises(BoxSizeError, + lambda: self.box.make_random_square_container(1)) + + def test_neq(self): + other = Box(self.ymin + 1, self.xmin, self.ymax, self.xmax) + self.assertTrue(self.box != other) + + def test_int(self): + other = Box( + float(self.ymin) + 0.01, float(self.xmin), float(self.ymax), + float(self.xmax)) + self.assertTrue(other.to_int() == self.box) + + def test_get_height(self): + height = self.ymax - self.ymin + self.assertEqual(self.box.get_height(), height) + + def test_get_width(self): + width = self.xmax - self.xmin + self.assertEqual(self.box.get_width(), width) + + def test_get_area(self): + area = self.box.get_height() * self.box.get_width() + self.assertEqual(self.box.get_area(), area) + + def test_rasterio_format(self): + rasterio_box = ((self.ymin, self.ymax), (self.xmin, self.xmax)) + self.assertEqual(self.box.rasterio_format(), rasterio_box) + + def test_tuple_format(self): + box_tuple = (0, 0, 2, 3) + output_box = self.box.tuple_format() + self.assertEqual(output_box, box_tuple) + + def test_shapely_format(self): + shapely_box = (self.xmin, self.ymin, self.xmax, self.ymax) + self.assertEqual(self.box.shapely_format(), shapely_box) + + def test_npbox_format(self): + self.assertEqual( + tuple(self.box.npbox_format()), self.box.tuple_format()) + self.assertEqual(self.box.npbox_format().dtype, np.float) + + def test_geojson_coordinates(self): + nw = [self.xmin, self.ymin] + ne = [self.xmin, self.ymax] + se = [self.xmax, self.ymax] + sw = [self.xmax, self.ymin] + geojson_coords = [nw, ne, se, sw, nw] + self.assertEqual(self.box.geojson_coordinates(), geojson_coords) + + def test_make_random_square_container(self): + size = 5 + nb_tests = 100 + for _ in range(nb_tests): + container = self.box.make_random_square_container(size) + self.assertEqual(container.get_width(), container.get_height()) + self.assertEqual(container.get_width(), size) + self.assertTrue(container.to_shapely().contains( + self.box.to_shapely())) + + def test_make_random_square_container_too_big(self): + size = 1 + with self.assertRaises(BoxSizeError): + self.box.make_random_square_container(size) + + def test_make_random_square(self): + window = Box(5, 5, 15, 15) + size = 5 + nb_tests = 100 + for _ in range(nb_tests): + box = window.make_random_square(size) + self.assertEqual(box.get_width(), box.get_height()) + self.assertEqual(box.get_width(), size) + self.assertTrue(window.to_shapely().contains(box.to_shapely())) + + def test_from_npbox(self): + npbox = np.array([self.ymin, self.xmin, self.ymax, self.xmax]) + output_box = Box.from_npbox(npbox) + self.assertEqual(output_box, self.box) + + def test_from_shapely(self): + shape = ShapelyBox(self.xmin, self.ymin, self.xmax, self.ymax) + output_box = Box.from_shapely(shape) + self.assertEqual(output_box, self.box) + + def test_to_shapely(self): + bounds = self.box.to_shapely().bounds + self.assertEqual((bounds[1], bounds[0], bounds[3], bounds[2]), + self.box.tuple_format()) + + def test_make_square(self): + square = Box(0, 0, 10, 10) + output_square = Box.make_square(0, 0, 10) + self.assertEqual(output_square, square) + self.assertEqual(output_square.get_width(), output_square.get_height()) + + def test_make_eroded(self): + max_extent = Box.make_square(0, 0, 10) + box = Box(1, 1, 3, 4) + buffer_size = erosion_size = 1 + eroded_box = box.make_buffer(buffer_size, max_extent) \ + .make_eroded(erosion_size) + self.assertEqual(eroded_box, box) + + def test_make_buffer(self): + buffer_size = 1 + max_extent = Box.make_square(0, 0, 3) + buffer_box = Box(0, 0, 3, 3) + output_buffer_box = self.box.make_buffer(buffer_size, max_extent) + self.assertEqual(output_buffer_box, buffer_box) + + buffer_size = 0.5 + max_extent = Box.make_square(0, 0, 5) + buffer_box = Box(0, 0, 3, 5) + output_buffer_box = self.box.make_buffer(buffer_size, max_extent) + self.assertEqual(output_buffer_box, buffer_box) + + def test_make_copy(self): + copy_box = self.box.make_copy() + self.assertIsNot(copy_box, self.box) + self.assertEqual(copy_box, self.box) + + def test_get_windows(self): + extent = Box(0, 0, 100, 100) + windows = list(extent.get_windows(10, 10)) + self.assertEqual(len(windows), 100) + + extent = Box(0, 0, 100, 100) + windows = list(extent.get_windows(10, 5)) + self.assertEqual(len(windows), 400) + + extent = Box(0, 0, 20, 20) + windows = set( + [window.tuple_format() for window in extent.get_windows(10, 10)]) + expected_windows = [ + Box.make_square(0, 0, 10), + Box.make_square(10, 0, 10), + Box.make_square(0, 10, 10), + Box.make_square(10, 10, 10) + ] + expected_windows = set( + [window.tuple_format() for window in expected_windows]) + self.assertSetEqual(windows, expected_windows) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/core/test_stats_analyzer.py b/tests_v2/core/test_stats_analyzer.py new file mode 100644 index 000000000..f6a419cb5 --- /dev/null +++ b/tests_v2/core/test_stats_analyzer.py @@ -0,0 +1,73 @@ +import unittest +import os + +import numpy as np + +from rastervision2.pipeline import rv_config +from rastervision2.core.raster_stats import RasterStats, chip_sz +from rastervision2.core.data import Scene +from rastervision2.core.analyzer import StatsAnalyzerConfig +from tests_v2.core.data.mock_raster_source import MockRasterSource + + +class TestStatsAnalyzer(unittest.TestCase): + def setUp(self): + self.tmp_dir = rv_config.get_tmp_dir() + + def tearDown(self): + self.tmp_dir.cleanup() + + def _test(self, is_random=False): + stats_uri = os.path.join(self.tmp_dir.name, 'stats.json') + scenes = [] + raster_sources = [] + imgs = [] + sample_prob = 0.5 + for i in range(3): + rs = MockRasterSource([0, 1, 2], 3) + img = np.zeros((600, 600, 3)) + img[:, :, 0] = 1 + i + img[:, :, 1] = 2 + i + img[:, :, 2] = 3 + i + if not is_random: + img[300:, 300:, :] = np.nan + + imgs.append(img) + rs.set_raster(img) + raster_sources.append(rs) + scenes.append(Scene(str(i), rs)) + + channel_vals = list(map(lambda x: np.expand_dims(x, axis=0), imgs)) + channel_vals = np.concatenate(channel_vals, axis=0) + channel_vals = np.transpose(channel_vals, [3, 0, 1, 2]) + channel_vals = np.reshape(channel_vals, (3, -1)) + exp_means = np.nanmean(channel_vals, axis=1) + exp_stds = np.nanstd(channel_vals, axis=1) + + analyzer_cfg = StatsAnalyzerConfig(output_uri=stats_uri, sample_prob=None) + if is_random: + analyzer_cfg = StatsAnalyzerConfig( + output_uri=stats_uri, sample_prob=sample_prob) + analyzer = analyzer_cfg.build() + analyzer.process(scenes, self.tmp_dir.name) + + stats = RasterStats.load(stats_uri) + np.testing.assert_array_almost_equal(stats.means, exp_means, decimal=3) + np.testing.assert_array_almost_equal(stats.stds, exp_stds, decimal=3) + if is_random: + for rs in raster_sources: + width = rs.get_extent().get_width() + height = rs.get_extent().get_height() + exp_num_chips = round( + ((width * height) / (chip_sz**2)) * sample_prob) + self.assertEqual(rs.mock._get_chip.call_count, exp_num_chips) + + def test_random(self): + self._test(is_random=True) + + def test_sliding(self): + self._test(is_random=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/data_files/0-aoi.geojson b/tests_v2/data_files/0-aoi.geojson new file mode 100644 index 000000000..b4f520628 --- /dev/null +++ b/tests_v2/data_files/0-aoi.geojson @@ -0,0 +1,43 @@ +{ + "type": "FeatureCollection", + "name": "1-aoi", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + } + }, + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27068776527476, + 36.19653624151811 + ], + [ + -115.26997101697017, + 36.19653108505549 + ], + [ + -115.26997710815654, + 36.19568441015065 + ], + [ + -115.27069385646112, + 36.19568956661327 + ], + [ + -115.27068776527476, + 36.19653624151811 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/0-gt-polygons.geojson b/tests_v2/data_files/0-gt-polygons.geojson new file mode 100644 index 000000000..369930d5b --- /dev/null +++ b/tests_v2/data_files/0-gt-polygons.geojson @@ -0,0 +1,191 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.26914477348328, + 36.19628708746127 + ], + [ + -115.26894897222517, + 36.19628708746127 + ], + [ + -115.26894897222517, + 36.19674596835801 + ], + [ + -115.26914477348328, + 36.19674596835801 + ], + [ + -115.26914477348328, + 36.19628708746127 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.2694746851921, + 36.19587798854514 + ], + [ + -115.2690589427948, + 36.19587798854514 + ], + [ + -115.2690589427948, + 36.19604682327752 + ], + [ + -115.2694746851921, + 36.19604682327752 + ], + [ + -115.2694746851921, + 36.19587798854514 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.26903748512267, + 36.19588881129532 + ], + [ + -115.26885777711868, + 36.19588881129532 + ], + [ + -115.26885777711868, + 36.196061975094665 + ], + [ + -115.26903748512267, + 36.196061975094665 + ], + [ + -115.26903748512267, + 36.19588881129532 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27055829763411, + 36.19575893819448 + ], + [ + -115.27038127183914, + 36.19575893819448 + ], + [ + -115.27038127183914, + 36.19594725412043 + ], + [ + -115.27055829763411, + 36.19594725412043 + ], + [ + -115.27055829763411, + 36.19575893819448 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27020424604414, + 36.19575027998011 + ], + [ + -115.2700325846672, + 36.19575027998011 + ], + [ + -115.2700325846672, + 36.19594725412043 + ], + [ + -115.27020424604414, + 36.19594725412043 + ], + [ + -115.27020424604414, + 36.19575027998011 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27036517858504, + 36.19640613700888 + ], + [ + -115.27009159326553, + 36.19640613700888 + ], + [ + -115.27009159326553, + 36.19662691932712 + ], + [ + -115.27036517858504, + 36.19662691932712 + ], + [ + -115.27036517858504, + 36.19640613700888 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/0-pred-polygons.geojson b/tests_v2/data_files/0-pred-polygons.geojson new file mode 100644 index 000000000..cf7fc6d17 --- /dev/null +++ b/tests_v2/data_files/0-pred-polygons.geojson @@ -0,0 +1,199 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.26914477348328, + 36.19628708746127 + ], + [ + -115.26894897222517, + 36.19628708746127 + ], + [ + -115.26894897222517, + 36.19674596835801 + ], + [ + -115.26914477348328, + 36.19674596835801 + ], + [ + -115.26914477348328, + 36.19628708746127 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.26903748512267, + 36.19588881129532 + ], + [ + -115.26885777711868, + 36.19588881129532 + ], + [ + -115.26885777711868, + 36.196061975094665 + ], + [ + -115.26903748512267, + 36.196061975094665 + ], + [ + -115.26903748512267, + 36.19588881129532 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27055829763411, + 36.19575893819448 + ], + [ + -115.27038127183914, + 36.19575893819448 + ], + [ + -115.27038127183914, + 36.19594725412043 + ], + [ + -115.27055829763411, + 36.19594725412043 + ], + [ + -115.27055829763411, + 36.19575893819448 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27020424604414, + 36.19575027998011 + ], + [ + -115.2700325846672, + 36.19575027998011 + ], + [ + -115.2700325846672, + 36.19594725412043 + ], + [ + -115.27020424604414, + 36.19594725412043 + ], + [ + -115.27020424604414, + 36.19575027998011 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.27036517858504, + 36.19640613700888 + ], + [ + -115.27009159326553, + 36.19640613700888 + ], + [ + -115.27009159326553, + 36.19662691932712 + ], + [ + -115.27036517858504, + 36.19662691932712 + ], + [ + -115.27036517858504, + 36.19640613700888 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 0 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.26969194412231, + 36.19664423556104 + ], + [ + -115.26987433433531, + 36.19641263061537 + ], + [ + -115.26945859193802, + 36.19641912422132 + ], + [ + -115.26969194412231, + 36.19664423556104 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/1-gt-polygons.geojson b/tests_v2/data_files/1-gt-polygons.geojson new file mode 100644 index 000000000..17efd501d --- /dev/null +++ b/tests_v2/data_files/1-gt-polygons.geojson @@ -0,0 +1,253 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23911744356155, + 36.21938363838411 + ], + [ + -115.23871511220932, + 36.21938363838411 + ], + [ + -115.23871511220932, + 36.21959570031398 + ], + [ + -115.23911744356155, + 36.21959570031398 + ], + [ + -115.23911744356155, + 36.21938363838411 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.2386400103569, + 36.2192581320732 + ], + [ + -115.2376878261566, + 36.2192581320732 + ], + [ + -115.2376878261566, + 36.21956973357793 + ], + [ + -115.2386400103569, + 36.21956973357793 + ], + [ + -115.2386400103569, + 36.2192581320732 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23915767669678, + 36.21812856621411 + ], + [ + -115.23877680301665, + 36.21812856621411 + ], + [ + -115.23877680301665, + 36.21845748362631 + ], + [ + -115.23915767669678, + 36.21845748362631 + ], + [ + -115.23915767669678, + 36.21812856621411 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23866415023805, + 36.21851374567675 + ], + [ + -115.23840665817261, + 36.21851374567675 + ], + [ + -115.23840665817261, + 36.2187344656377 + ], + [ + -115.23866415023805, + 36.2187344656377 + ], + [ + -115.23866415023805, + 36.21851374567675 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23746252059937, + 36.218037680895826 + ], + [ + -115.23730158805847, + 36.218037680895826 + ], + [ + -115.23730158805847, + 36.21818050063428 + ], + [ + -115.23746252059937, + 36.21818050063428 + ], + [ + -115.23746252059937, + 36.218037680895826 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23794531822205, + 36.21876476048567 + ], + [ + -115.23758590221405, + 36.21876476048567 + ], + [ + -115.23758590221405, + 36.21916292025475 + ], + [ + -115.23794531822205, + 36.21916292025475 + ], + [ + -115.23794531822205, + 36.21876476048567 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23796677589415, + 36.21848345073153 + ], + [ + -115.2377200126648, + 36.21848345073153 + ], + [ + -115.2377200126648, + 36.21869551510161 + ], + [ + -115.23796677589415, + 36.21869551510161 + ], + [ + -115.23796677589415, + 36.21848345073153 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.2386963367462, + 36.21813289408377 + ], + [ + -115.23839592933655, + 36.21813289408377 + ], + [ + -115.23839592933655, + 36.21832764797091 + ], + [ + -115.2386963367462, + 36.21832764797091 + ], + [ + -115.2386963367462, + 36.21813289408377 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/1-pred-polygons.geojson b/tests_v2/data_files/1-pred-polygons.geojson new file mode 100644 index 000000000..a98987473 --- /dev/null +++ b/tests_v2/data_files/1-pred-polygons.geojson @@ -0,0 +1,236 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23911744356155, + 36.21938363838411 + ], + [ + -115.23871511220932, + 36.21938363838411 + ], + [ + -115.23871511220932, + 36.21959570031398 + ], + [ + -115.23911744356155, + 36.21959570031398 + ], + [ + -115.23911744356155, + 36.21938363838411 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23866415023805, + 36.21851374567675 + ], + [ + -115.23840665817261, + 36.21851374567675 + ], + [ + -115.23840665817261, + 36.2187344656377 + ], + [ + -115.23866415023805, + 36.2187344656377 + ], + [ + -115.23866415023805, + 36.21851374567675 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23746252059937, + 36.218037680895826 + ], + [ + -115.23730158805847, + 36.218037680895826 + ], + [ + -115.23730158805847, + 36.21818050063428 + ], + [ + -115.23746252059937, + 36.21818050063428 + ], + [ + -115.23746252059937, + 36.218037680895826 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23794531822205, + 36.21876476048567 + ], + [ + -115.23758590221405, + 36.21876476048567 + ], + [ + -115.23758590221405, + 36.21916292025475 + ], + [ + -115.23794531822205, + 36.21916292025475 + ], + [ + -115.23794531822205, + 36.21876476048567 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.23796677589415, + 36.21848345073153 + ], + [ + -115.2377200126648, + 36.21848345073153 + ], + [ + -115.2377200126648, + 36.21869551510161 + ], + [ + -115.23796677589415, + 36.21869551510161 + ], + [ + -115.23796677589415, + 36.21848345073153 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.2386963367462, + 36.21813289408377 + ], + [ + -115.23839592933655, + 36.21813289408377 + ], + [ + -115.23839592933655, + 36.21832764797091 + ], + [ + -115.2386963367462, + 36.21832764797091 + ], + [ + -115.2386963367462, + 36.21813289408377 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "class_id": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -115.2389645576477, + 36.218310336533904 + ], + [ + -115.23906648159026, + 36.21813289408377 + ], + [ + -115.23880362510681, + 36.218080959632 + ], + [ + -115.2387446165085, + 36.218249746474235 + ], + [ + -115.2389645576477, + 36.218310336533904 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/2-gt-polygons.geojson b/tests_v2/data_files/2-gt-polygons.geojson new file mode 100644 index 000000000..8a6d82f5f --- /dev/null +++ b/tests_v2/data_files/2-gt-polygons.geojson @@ -0,0 +1,67 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 31.129524707794186, + 29.975011149702052 + ], + [ + 31.131863594055172, + 29.975011149702052 + ], + [ + 31.131863594055172, + 29.97724163265764 + ], + [ + 31.129524707794186, + 29.97724163265764 + ], + [ + 31.129524707794186, + 29.975011149702052 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 31.137126088142395, + 29.975104087491815 + ], + [ + 31.13801121711731, + 29.975104087491815 + ], + [ + 31.13801121711731, + 29.975452603428955 + ], + [ + 31.137126088142395, + 29.975452603428955 + ], + [ + 31.137126088142395, + 29.975104087491815 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/2-pred-polygons.geojson b/tests_v2/data_files/2-pred-polygons.geojson new file mode 100644 index 000000000..a250b9fd2 --- /dev/null +++ b/tests_v2/data_files/2-pred-polygons.geojson @@ -0,0 +1,67 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 31.132979393005368, + 29.97807805085361 + ], + [ + 31.135404109954834, + 29.97807805085361 + ], + [ + 31.135404109954834, + 29.980234118612632 + ], + [ + 31.132979393005368, + 29.980234118612632 + ], + [ + 31.132979393005368, + 29.97807805085361 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 31.129524707794186, + 29.975011149702052 + ], + [ + 31.131863594055172, + 29.975011149702052 + ], + [ + 31.131863594055172, + 29.97724163265764 + ], + [ + 31.129524707794186, + 29.97724163265764 + ], + [ + 31.129524707794186, + 29.975011149702052 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/evaluator/cc-label-aoi.json b/tests_v2/data_files/evaluator/cc-label-aoi.json new file mode 100644 index 000000000..9029471ba --- /dev/null +++ b/tests_v2/data_files/evaluator/cc-label-aoi.json @@ -0,0 +1,36 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048480153083801, + 52.40971027985026 + ], + [ + 13.047691583633423, + 52.40968410101556 + ], + [ + 13.047696948051453, + 52.409440309871265 + ], + [ + 13.048501610755919, + 52.40947630596393 + ], + [ + 13.048480153083801, + 52.40971027985026 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/evaluator/cc-label-filtered.json b/tests_v2/data_files/evaluator/cc-label-filtered.json new file mode 100644 index 000000000..9b4378955 --- /dev/null +++ b/tests_v2/data_files/evaluator/cc-label-filtered.json @@ -0,0 +1,385 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "scores": [ + 0.009182669222354889, + 0.053312476724386215, + 0.9375048875808716 + ], + "class_name": "background" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048166982788103, + 52.40967025235487 + ], + [ + 13.048170950365797, + 52.40958040125634 + ], + [ + 13.048317875729522, + 52.40958282750112 + ], + [ + 13.04831390845003, + 52.40967267860747 + ], + [ + 13.048166982788103, + 52.40967025235487 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.9955767393112183, + 0.003868062049150467, + 0.0005552001530304551 + ], + "class_name": "car" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.047873131526996, + 52.40966539930218 + ], + [ + 13.047877099701086, + 52.40957554821931 + ], + [ + 13.048024025022983, + 52.40957797482907 + ], + [ + 13.048020057147093, + 52.40966782591979 + ], + [ + 13.047873131526996, + 52.40966539930218 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.9934515953063965, + 0.002841504756361246, + 0.003706953953951597 + ], + "class_name": "car" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048024025022983, + 52.40957797482907 + ], + [ + 13.048027992877959, + 52.40948812373675 + ], + [ + 13.048174917922575, + 52.40949055015622 + ], + [ + 13.048170950365797, + 52.40958040125634 + ], + [ + 13.048024025022983, + 52.40957797482907 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.0021566825453191996, + 0.001589148654602468, + 0.9962542057037354 + ], + "class_name": "background" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.04831390845003, + 52.40967267860747 + ], + [ + 13.048317875729522, + 52.40958282750112 + ], + [ + 13.04846480111416, + 52.4095852535634 + ], + [ + 13.04846083413287, + 52.40967510467758 + ], + [ + 13.04831390845003, + 52.40967267860747 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.010937422513961792, + 0.983174741268158, + 0.005887891165912151 + ], + "class_name": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048020057147093, + 52.40966782591979 + ], + [ + 13.048024025022983, + 52.40957797482907 + ], + [ + 13.048170950365797, + 52.40958040125634 + ], + [ + 13.048166982788103, + 52.40967025235487 + ], + [ + 13.048020057147093, + 52.40966782591979 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.00039659003959968686, + 0.00930043589323759, + 0.9903029799461365 + ], + "class_name": "background" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048170950365797, + 52.40958040125634 + ], + [ + 13.048174917922575, + 52.40949055015622 + ], + [ + 13.0483218429881, + 52.409492976393175 + ], + [ + 13.048317875729522, + 52.40958282750112 + ], + [ + 13.048170950365797, + 52.40958040125634 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.9986116886138916, + 0.00022434288985095918, + 0.0011639182921499014 + ], + "class_name": "car" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.047730174400108, + 52.40957312142703 + ], + [ + 13.047734142851478, + 52.40948327035038 + ], + [ + 13.047881067854261, + 52.40948569713481 + ], + [ + 13.047877099701086, + 52.40957554821931 + ], + [ + 13.047730174400108, + 52.40957312142703 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.989732563495636, + 0.0032721608877182007, + 0.00699527096003294 + ], + "class_name": "car" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.047877099701086, + 52.40957554821931 + ], + [ + 13.047881067854261, + 52.40948569713481 + ], + [ + 13.048027992877959, + 52.40948812373675 + ], + [ + 13.048024025022983, + 52.40957797482907 + ], + [ + 13.047877099701086, + 52.40957554821931 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.0006972793489694595, + 0.0004679459671024233, + 0.9988347887992859 + ], + "class_name": "background" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.047726205927816, + 52.4096629725021 + ], + [ + 13.047730174400108, + 52.40957312142703 + ], + [ + 13.047877099701086, + 52.40957554821931 + ], + [ + 13.047873131526996, + 52.40966539930218 + ], + [ + 13.047726205927816, + 52.4096629725021 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "scores": [ + 0.019290348514914513, + 0.9737153053283691, + 0.006994331255555153 + ], + "class_name": "building" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 13.048317875729522, + 52.40958282750112 + ], + [ + 13.0483218429881, + 52.409492976393175 + ], + [ + 13.04846876807454, + 52.40949540244763 + ], + [ + 13.04846480111416, + 52.4095852535634 + ], + [ + 13.048317875729522, + 52.40958282750112 + ] + ] + ] + } + } + ] +} diff --git a/tests_v2/data_files/evaluator/cc-label-full.json b/tests_v2/data_files/evaluator/cc-label-full.json new file mode 100644 index 000000000..ca1d45669 --- /dev/null +++ b/tests_v2/data_files/evaluator/cc-label-full.json @@ -0,0 +1,256 @@ +{ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "properties": { + "scores": [0.009182669222354889, 0.053312476724386215, 0.9375048875808716], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.048166982788103, 52.40967025235487], + [13.048170950365797, 52.40958040125634], + [13.048317875729522, 52.40958282750112], + [13.04831390845003, 52.40967267860747], + [13.048166982788103, 52.40967025235487] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.004562287125736475, 0.0025018006563186646, 0.9929359555244446], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.04846480111416, 52.4095852535634], + [13.04846876807454, 52.40949540244763], + [13.04861569318189, 52.409497828319594], + [13.04861172651971, 52.409587679443185], + [13.04846480111416, 52.4095852535634] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.9955767393112183, 0.003868062049150467, 0.0005552001530304551], + "class_name": "car" + }, + "geometry": { + "coordinates": [ + [ + [13.047873131526996, 52.40966539930218], + [13.047877099701086, 52.40957554821931], + [13.048024025022983, 52.40957797482907], + [13.048020057147093, 52.40966782591979], + [13.047873131526996, 52.40966539930218] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.9934515953063965, 0.002841504756361246, 0.003706953953951597], + "class_name": "car" + }, + "geometry": { + "coordinates": [ + [ + [13.048024025022983, 52.40957797482907], + [13.048027992877959, 52.40948812373675], + [13.048174917922575, 52.40949055015622], + [13.048170950365797, 52.40958040125634], + [13.048024025022983, 52.40957797482907] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.0021566825453191996, 0.001589148654602468, 0.9962542057037354], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.04831390845003, 52.40967267860747], + [13.048317875729522, 52.40958282750112], + [13.04846480111416, 52.4095852535634], + [13.04846083413287, 52.40967510467758], + [13.04831390845003, 52.40967267860747] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.046653617173433304, 0.11211998760700226, 0.8412263989448547], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.048607759836617, 52.40967753056518], + [13.04861172651971, 52.409587679443185], + [13.048758651946166, 52.409590105140474], + [13.048754685561278, 52.40967995627029], + [13.048607759836617, 52.40967753056518] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.010937422513961792, 0.983174741268158, 0.005887891165912151], + "class_name": "building" + }, + "geometry": { + "coordinates": [ + [ + [13.048020057147093, 52.40966782591979], + [13.048024025022983, 52.40957797482907], + [13.048170950365797, 52.40958040125634], + [13.048166982788103, 52.40967025235487], + [13.048020057147093, 52.40966782591979] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.02746420167386532, 0.0202524121850729, 0.9522833824157715], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.04861172651971, 52.409587679443185], + [13.04861569318189, 52.409497828319594], + [13.04876261831015, 52.40950025400906], + [13.048758651946166, 52.409590105140474], + [13.04861172651971, 52.409587679443185] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.00039659003959968686, 0.00930043589323759, 0.9903029799461365], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.048170950365797, 52.40958040125634], + [13.048174917922575, 52.40949055015622], + [13.0483218429881, 52.409492976393175], + [13.048317875729522, 52.40958282750112], + [13.048170950365797, 52.40958040125634] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.9986116886138916, 0.00022434288985095918, 0.0011639182921499014], + "class_name": "car" + }, + "geometry": { + "coordinates": [ + [ + [13.047730174400108, 52.40957312142703], + [13.047734142851478, 52.40948327035038], + [13.047881067854261, 52.40948569713481], + [13.047877099701086, 52.40957554821931], + [13.047730174400108, 52.40957312142703] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.989732563495636, 0.0032721608877182007, 0.00699527096003294], + "class_name": "car" + }, + "geometry": { + "coordinates": [ + [ + [13.047877099701086, 52.40957554821931], + [13.047881067854261, 52.40948569713481], + [13.048027992877959, 52.40948812373675], + [13.048024025022983, 52.40957797482907], + [13.047877099701086, 52.40957554821931] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.00015513361722696573, 0.000368093402357772, 0.9994768500328064], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.04846083413287, 52.40967510467758], + [13.04846480111416, 52.4095852535634], + [13.04861172651971, 52.409587679443185], + [13.048607759836617, 52.40967753056518], + [13.04846083413287, 52.40967510467758] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.0006972793489694595, 0.0004679459671024233, 0.9988347887992859], + "class_name": "background" + }, + "geometry": { + "coordinates": [ + [ + [13.047726205927816, 52.4096629725021], + [13.047730174400108, 52.40957312142703], + [13.047877099701086, 52.40957554821931], + [13.047873131526996, 52.40966539930218], + [13.047726205927816, 52.4096629725021] + ] + ], + "type": "Polygon" + } + }, { + "type": "Feature", + "properties": { + "scores": [0.019290348514914513, 0.9737153053283691, 0.006994331255555153], + "class_name": "building" + }, + "geometry": { + "coordinates": [ + [ + [13.048317875729522, 52.40958282750112], + [13.0483218429881, 52.409492976393175], + [13.04846876807454, 52.40949540244763], + [13.04846480111416, 52.4095852535634], + [13.048317875729522, 52.40958282750112] + ] + ], + "type": "Polygon" + } + }] +} diff --git a/tests_v2/data_files/evaluator/cc-label-img-blank.tif b/tests_v2/data_files/evaluator/cc-label-img-blank.tif new file mode 100644 index 000000000..cb4cf0e1c Binary files /dev/null and b/tests_v2/data_files/evaluator/cc-label-img-blank.tif differ diff --git a/tests_v2/data_files/expected-eval.json b/tests_v2/data_files/expected-eval.json new file mode 100644 index 000000000..52bbdf293 --- /dev/null +++ b/tests_v2/data_files/expected-eval.json @@ -0,0 +1,172 @@ +{ + "overall": [ + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + 50, + 50, + 0 + ], + "class_id": 0, + "class_name": "one" + }, + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + 50, + 50, + 0 + ], + "class_id": 1, + "class_name": "two" + }, + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 200, + "conf_mat": [ + [ + 0, + 0, + 0 + ], + [ + 50, + 50, + 0 + ], + [ + 50, + 50, + 0 + ] + ], + "class_id": null, + "class_name": "average" + } + ], + "per_scene": { + "0": [ + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + 50, + 50, + 0 + ], + "class_id": 0, + "class_name": "one" + }, + { + "precision": 0, + "recall": null, + "f1": null, + "count_error": 50, + "gt_count": 0, + "conf_mat": [ + 0, + 0, + 0 + ], + "class_id": 1, + "class_name": "two" + }, + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + [ + 0, + 0, + 0 + ], + [ + 50, + 50, + 0 + ], + [ + 0, + 0, + 0 + ] + ], + "class_id": null, + "class_name": "average" + } + ], + "1": [ + { + "precision": 0, + "recall": null, + "f1": null, + "count_error": 50, + "gt_count": 0, + "conf_mat": [ + 0, + 0, + 0 + ], + "class_id": 0, + "class_name": "one" + }, + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + 50, + 50, + 0 + ], + "class_id": 1, + "class_name": "two" + }, + { + "precision": 1, + "recall": 0.5, + "f1": 0.6666666666666666, + "count_error": 50, + "gt_count": 100, + "conf_mat": [ + [ + 0, + 0, + 0 + ], + [ + 0, + 0, + 0 + ], + [ + 50, + 50, + 0 + ] + ], + "class_id": null, + "class_name": "average" + } + ] + } +} diff --git a/tests_v2/data_files/expected-vector-eval-with-aoi.json b/tests_v2/data_files/expected-vector-eval-with-aoi.json new file mode 100644 index 000000000..e678019bb --- /dev/null +++ b/tests_v2/data_files/expected-vector-eval-with-aoi.json @@ -0,0 +1,44 @@ +{ + "per_scene": { + "0": [ + { + "recall": 1.0, + "class_id": 0, + "gt_count": 3, + "precision": 1.0, + "f1": 1.0, + "class_name": "vector-polygons-one", + "count_error": 0 + }, + { + "recall": 1.0, + "class_id": null, + "gt_count": 3, + "precision": 1.0, + "f1": 1.0, + "class_name": "average", + "count_error": 0 + } + ] + }, + "overall": [ + { + "recall": 1.0, + "class_id": 0, + "gt_count": 3, + "precision": 1.0, + "f1": 1.0, + "class_name": "vector-polygons-one", + "count_error": 0 + }, + { + "recall": 1.0, + "class_id": null, + "gt_count": 3, + "precision": 1.0, + "f1": 1.0, + "class_name": "average", + "count_error": 0 + } + ] +} diff --git a/tests_v2/data_files/expected-vector-eval.json b/tests_v2/data_files/expected-vector-eval.json new file mode 100644 index 000000000..24ba03411 --- /dev/null +++ b/tests_v2/data_files/expected-vector-eval.json @@ -0,0 +1,73 @@ +{ + "overall": [ + { + "precision": 0.8333333333333334, + "recall": 0.8333333333333334, + "f1": 0.8333333333333334, + "count_error": 2, + "gt_count": 6, + "class_id": 0, + "class_name": "vector-polygons-one" + }, + { + "precision": 0.8571428571428571, + "recall": 0.75, + "f1": 0.7999999999999999, + "count_error": 3, + "gt_count": 8, + "class_id": 1, + "class_name": "vector-polygons-two" + }, + { + "precision": 0.846938775510204, + "recall": 0.7857142857142857, + "f1": 0.8142857142857143, + "count_error": 2.571428571428571, + "gt_count": 14, + "class_id": null, + "class_name": "average" + } + ], + "per_scene": { + "0": [ + { + "precision": 0.8333333333333334, + "recall": 0.8333333333333334, + "f1": 0.8333333333333334, + "count_error": 2, + "gt_count": 6, + "class_id": 0, + "class_name": "vector-polygons-one" + }, + { + "precision": 0.8333333333333334, + "recall": 0.8333333333333334, + "f1": 0.8333333333333334, + "count_error": 2, + "gt_count": 6, + "class_id": null, + "class_name": "average" + } + ], + "1": [ + { + "precision": 0.8571428571428571, + "recall": 0.75, + "f1": 0.7999999999999999, + "count_error": 3, + "gt_count": 8, + "class_id": 1, + "class_name": "vector-polygons-two" + }, + { + "precision": 0.8571428571428571, + "recall": 0.75, + "f1": 0.7999999999999999, + "count_error": 3, + "gt_count": 8, + "class_id": null, + "class_name": "average" + } + ] + } +} diff --git a/tests_v2/data_files/ones.tif b/tests_v2/data_files/ones.tif new file mode 100644 index 000000000..90fbe2589 Binary files /dev/null and b/tests_v2/data_files/ones.tif differ diff --git a/tests_v2/data_files/small-rgb-tile.tif b/tests_v2/data_files/small-rgb-tile.tif new file mode 100644 index 000000000..209ecc5ef Binary files /dev/null and b/tests_v2/data_files/small-rgb-tile.tif differ diff --git a/tests_v2/data_files/small-uint16-tile.tif b/tests_v2/data_files/small-uint16-tile.tif new file mode 100644 index 000000000..d0c8e7bfa Binary files /dev/null and b/tests_v2/data_files/small-uint16-tile.tif differ diff --git a/tests_v2/pipeline/__init__.py b/tests_v2/pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_v2/pipeline/test_config.py b/tests_v2/pipeline/test_config.py similarity index 100% rename from test_v2/pipeline/test_config.py rename to tests_v2/pipeline/test_config.py diff --git a/tests_v2/pipeline/test_file_system.py b/tests_v2/pipeline/test_file_system.py new file mode 100644 index 000000000..91471927f --- /dev/null +++ b/tests_v2/pipeline/test_file_system.py @@ -0,0 +1,519 @@ +import os +import unittest +from unittest.mock import patch +import datetime +import gzip + +import boto3 +from moto import mock_s3 + +from rastervision2.pipeline.file_system import ( + file_to_str, str_to_file, download_if_needed, upload_or_copy, + make_dir, get_local_path, file_exists, sync_from_dir, sync_to_dir, list_paths, + get_cached_file, NotReadableError, NotWritableError, FileSystem) +from rastervision2.pipeline import rv_config + +LOREM = """ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut + enim ad minim veniam, quis nostrud exercitation ullamco + laboris nisi ut aliquip ex ea commodo consequat. Duis aute + irure dolor in reprehenderit in voluptate velit esse cillum + dolore eu fugiat nulla pariatur. Excepteur sint occaecat + cupidatat non proident, sunt in culpa qui officia deserunt + mollit anim id est laborum. """ + + +class TestMakeDir(unittest.TestCase): + def setUp(self): + self.lorem = LOREM + + # Mock S3 bucket + self.mock_s3 = mock_s3() + self.mock_s3.start() + self.s3 = boto3.client('s3') + self.bucket_name = 'mock_bucket' + self.s3.create_bucket(Bucket=self.bucket_name) + + # temporary directory + self.tmp_dir = rv_config.get_tmp_dir() + + def tearDown(self): + self.tmp_dir.cleanup() + self.mock_s3.stop() + + def test_default_args(self): + dir = os.path.join(self.tmp_dir.name, 'hello') + make_dir(dir) + self.assertTrue(os.path.isdir(dir)) + + def test_file_exists_local_true(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + str_to_file(self.lorem, path) + + self.assertTrue(file_exists(path)) + + def test_file_exists_local_false(self): + path = os.path.join(self.tmp_dir.name, 'hello', 'hello.txt') + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + self.assertFalse(file_exists(path)) + + def test_file_exists_s3_true(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + str_to_file(self.lorem, path) + + s3_path = 's3://{}/lorem.txt'.format(self.bucket_name) + upload_or_copy(path, s3_path) + + self.assertTrue(file_exists(s3_path)) + + def test_file_exists_s3_false(self): + s3_path = 's3://{}/hello.txt'.format(self.bucket_name) + self.assertFalse(file_exists(s3_path)) + + def test_check_empty(self): + path = os.path.join(self.tmp_dir.name, 'hello', 'hello.txt') + dir = os.path.dirname(path) + str_to_file('hello', path) + + make_dir(dir, check_empty=False) + with self.assertRaises(Exception): + make_dir(dir, check_empty=True) + + def test_force_empty(self): + path = os.path.join(self.tmp_dir.name, 'hello', 'hello.txt') + dir = os.path.dirname(path) + str_to_file('hello', path) + + make_dir(dir, force_empty=False) + self.assertTrue(os.path.isfile(path)) + make_dir(dir, force_empty=True) + is_empty = len(os.listdir(dir)) == 0 + self.assertTrue(is_empty) + + def test_use_dirname(self): + path = os.path.join(self.tmp_dir.name, 'hello', 'hello.txt') + dir = os.path.dirname(path) + make_dir(path, use_dirname=True) + self.assertTrue(os.path.isdir(dir)) + + +class TestGetLocalPath(unittest.TestCase): + def test_local(self): + download_dir = '/download_dir' + uri = '/my/file.txt' + path = get_local_path(uri, download_dir) + self.assertEqual(path, uri) + + def test_s3(self): + download_dir = '/download_dir' + uri = 's3://bucket/my/file.txt' + path = get_local_path(uri, download_dir) + self.assertEqual(path, '/download_dir/s3/bucket/my/file.txt') + + def test_http(self): + download_dir = '/download_dir' + uri = 'http://bucket/my/file.txt' + path = get_local_path(uri, download_dir) + self.assertEqual(path, '/download_dir/http/bucket/my/file.txt') + + # simulate a zxy tile URI + uri = 'http://bucket/10/25/53?auth=426753' + path = get_local_path(uri, download_dir) + self.assertEqual(path, '/download_dir/http/bucket/10/25/53') + + +class TestFileToStr(unittest.TestCase): + """Test file_to_str and str_to_file.""" + + def setUp(self): + # Setup mock S3 bucket. + self.mock_s3 = mock_s3() + self.mock_s3.start() + self.s3 = boto3.client('s3') + self.bucket_name = 'mock_bucket' + self.s3.create_bucket(Bucket=self.bucket_name) + + self.content_str = 'hello' + self.file_name = 'hello.txt' + self.s3_path = 's3://{}/{}'.format(self.bucket_name, self.file_name) + + self.tmp_dir = rv_config.get_tmp_dir() + self.local_path = os.path.join(self.tmp_dir.name, self.file_name) + + def tearDown(self): + self.tmp_dir.cleanup() + self.mock_s3.stop() + + def test_file_to_str_local(self): + str_to_file(self.content_str, self.local_path) + content_str = file_to_str(self.local_path) + self.assertEqual(self.content_str, content_str) + + wrong_path = '/wrongpath/x.txt' + with self.assertRaises(NotReadableError): + file_to_str(wrong_path) + + def test_file_to_str_s3(self): + wrong_path = 's3://wrongpath/x.txt' + + with self.assertRaises(NotWritableError): + str_to_file(self.content_str, wrong_path) + + str_to_file(self.content_str, self.s3_path) + content_str = file_to_str(self.s3_path) + self.assertEqual(self.content_str, content_str) + + with self.assertRaises(NotReadableError): + file_to_str(wrong_path) + + +class TestDownloadIfNeeded(unittest.TestCase): + """Test download_if_needed and upload_or_copy and str_to_file.""" + + def setUp(self): + # Setup mock S3 bucket. + self.mock_s3 = mock_s3() + self.mock_s3.start() + self.s3 = boto3.client('s3') + self.bucket_name = 'mock_bucket' + self.s3.create_bucket(Bucket=self.bucket_name) + + self.content_str = 'hello' + self.file_name = 'hello.txt' + self.s3_path = 's3://{}/{}'.format(self.bucket_name, self.file_name) + + self.tmp_dir = rv_config.get_tmp_dir() + self.local_path = os.path.join(self.tmp_dir.name, self.file_name) + + def tearDown(self): + self.tmp_dir.cleanup() + self.mock_s3.stop() + + def test_download_if_needed_local(self): + with self.assertRaises(NotReadableError): + file_to_str(self.local_path) + + str_to_file(self.content_str, self.local_path) + upload_or_copy(self.local_path, self.local_path) + local_path = download_if_needed(self.local_path, self.tmp_dir.name) + self.assertEqual(local_path, self.local_path) + + def test_download_if_needed_s3(self): + with self.assertRaises(NotReadableError): + file_to_str(self.s3_path) + + str_to_file(self.content_str, self.local_path) + upload_or_copy(self.local_path, self.s3_path) + local_path = download_if_needed(self.s3_path, self.tmp_dir.name) + content_str = file_to_str(local_path) + self.assertEqual(self.content_str, content_str) + + wrong_path = 's3://wrongpath/x.txt' + with self.assertRaises(NotWritableError): + upload_or_copy(local_path, wrong_path) + + +class TestS3Misc(unittest.TestCase): + def setUp(self): + self.lorem = LOREM + + # Mock S3 bucket + self.mock_s3 = mock_s3() + self.mock_s3.start() + self.s3 = boto3.client('s3') + self.bucket_name = 'mock_bucket' + self.s3.create_bucket(Bucket=self.bucket_name) + + # temporary directory + self.tmp_dir = rv_config.get_tmp_dir() + + def tearDown(self): + self.tmp_dir.cleanup() + self.mock_s3.stop() + + def test_last_modified_s3(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum1.txt') + s3_path = 's3://{}/lorem1.txt'.format(self.bucket_name) + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + fs = FileSystem.get_file_system(s3_path, 'r') + + str_to_file(self.lorem, path) + upload_or_copy(path, s3_path) + stamp = fs.last_modified(s3_path) + + self.assertTrue(isinstance(stamp, datetime.datetime)) + + def test_list_paths_s3(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + s3_path = 's3://{}/xxx/lorem.txt'.format(self.bucket_name) + s3_directory = 's3://{}/xxx/'.format(self.bucket_name) + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + str_to_file(self.lorem, path) + upload_or_copy(path, s3_path) + + list_paths(s3_directory) + self.assertEqual(len(list_paths(s3_directory)), 1) + + def test_file_exists(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + s3_path = 's3://{}/xxx/lorem.txt'.format(self.bucket_name) + s3_path_prefix = 's3://{}/xxx/lorem'.format(self.bucket_name) + s3_directory = 's3://{}/xxx/'.format(self.bucket_name) + make_dir(path, check_empty=False, use_dirname=True) + + str_to_file(self.lorem, path) + upload_or_copy(path, s3_path) + + self.assertTrue(file_exists(s3_directory, include_dir=True)) + self.assertTrue(file_exists(s3_path, include_dir=False)) + self.assertFalse(file_exists(s3_path_prefix, include_dir=True)) + self.assertFalse(file_exists(s3_directory, include_dir=False)) + self.assertFalse( + file_exists(s3_directory + 'NOTPOSSIBLE', include_dir=False)) + + +class TestLocalMisc(unittest.TestCase): + def setUp(self): + self.lorem = LOREM + self.tmp_dir = rv_config.get_tmp_dir() + + def tearDown(self): + self.tmp_dir.cleanup() + + def test_bytes_local(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + expected = bytes([0x00, 0x01, 0x02]) + fs = FileSystem.get_file_system(path, 'r') + + fs.write_bytes(path, expected) + actual = fs.read_bytes(path) + + self.assertEqual(actual, expected) + + def test_bytes_local_false(self): + path = os.path.join(self.tmp_dir.name, 'xxx') + fs = FileSystem.get_file_system(path, 'r') + self.assertRaises(NotReadableError, lambda: fs.read_bytes(path)) + + def test_sync_from_dir_local(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + src = os.path.dirname(path) + dst = os.path.join(self.tmp_dir.name, 'xxx') + make_dir(src, check_empty=False) + make_dir(dst, check_empty=False) + + fs = FileSystem.get_file_system(path, 'r') + fs.write_bytes(path, bytes([0x00, 0x01])) + sync_from_dir(src, dst, delete=True) + + self.assertEqual(len(list_paths(dst)), 1) + + def test_sync_from_dir_noop_local(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + src = os.path.join(self.tmp_dir.name, 'lorem') + make_dir(src, check_empty=False) + + fs = FileSystem.get_file_system(src, 'r') + fs.write_bytes(path, bytes([0x00, 0x01])) + sync_from_dir(src, src, delete=True) + + self.assertEqual(len(list_paths(src)), 1) + + def test_sync_to_dir_local(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + src = os.path.dirname(path) + dst = os.path.join(self.tmp_dir.name, 'xxx') + make_dir(src, check_empty=False) + make_dir(dst, check_empty=False) + + fs = FileSystem.get_file_system(path, 'r') + fs.write_bytes(path, bytes([0x00, 0x01])) + sync_to_dir(src, dst, delete=True) + + self.assertEqual(len(list_paths(dst)), 1) + + def test_copy_to_local(self): + path1 = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + path2 = os.path.join(self.tmp_dir.name, 'yyy', 'ipsum.txt') + dir1 = os.path.dirname(path1) + dir2 = os.path.dirname(path2) + make_dir(dir1, check_empty=False) + make_dir(dir2, check_empty=False) + + str_to_file(self.lorem, path1) + + upload_or_copy(path1, path2) + self.assertEqual(len(list_paths(dir2)), 1) + + def test_last_modified(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum1.txt') + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + fs = FileSystem.get_file_system(path, 'r') + + str_to_file(self.lorem, path) + stamp = fs.last_modified(path) + + self.assertTrue(isinstance(stamp, datetime.datetime)) + + def test_file_exists(self): + fs = FileSystem.get_file_system(self.tmp_dir.name, 'r') + + path1 = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + dir1 = os.path.dirname(path1) + make_dir(dir1, check_empty=False) + + str_to_file(self.lorem, path1) + + self.assertTrue(fs.file_exists(dir1, include_dir=True)) + self.assertTrue(fs.file_exists(path1, include_dir=False)) + self.assertFalse(fs.file_exists(dir1, include_dir=False)) + self.assertFalse( + fs.file_exists(dir1 + 'NOTPOSSIBLE', include_dir=False)) + + +class TestHttpMisc(unittest.TestCase): + def setUp(self): + self.lorem = LOREM + self.tmp_dir = rv_config.get_tmp_dir() + + def tearDown(self): + self.tmp_dir.cleanup() + + def test_file_exists_http_true(self): + http_path = ('https://raw.githubusercontent.com/tensorflow/models/' + '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md') + self.assertTrue(file_exists(http_path)) + + def test_file_exists_http_false(self): + http_path = ('https://raw.githubusercontent.com/tensorflow/models/' + '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/XXX') + self.assertFalse(file_exists(http_path)) + + def test_write_str_http(self): + self.assertRaises(NotWritableError, + lambda: str_to_file('xxx', 'http://localhost/')) + + def test_sync_to_http(self): + src = self.tmp_dir.name + dst = 'http://localhost/' + self.assertRaises(NotWritableError, lambda: sync_to_dir(src, dst)) + + def test_sync_from_http(self): + src = 'http://localhost/' + dst = self.tmp_dir.name + self.assertRaises(NotReadableError, lambda: sync_from_dir(src, dst)) + + def test_copy_to_http(self): + path = os.path.join(self.tmp_dir.name, 'lorem', 'ipsum.txt') + dst = 'http://localhost/' + directory = os.path.dirname(path) + make_dir(directory, check_empty=False) + + str_to_file(self.lorem, path) + + self.assertRaises(NotWritableError, lambda: upload_or_copy(path, dst)) + os.remove(path) + + def test_copy_from_http(self): + http_path = ('https://raw.githubusercontent.com/tensorflow/models/' + '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md') + expected = os.path.join( + self.tmp_dir.name, 'http', 'raw.githubusercontent.com', + 'tensorflow/models', + '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md') + download_if_needed(http_path, self.tmp_dir.name) + + self.assertTrue(file_exists(expected)) + os.remove(expected) + + def test_last_modified_http(self): + uri = 'http://localhost/' + fs = FileSystem.get_file_system(uri, 'r') + self.assertEqual(fs.last_modified(uri), None) + + def test_write_bytes_http(self): + uri = 'http://localhost/' + fs = FileSystem.get_file_system(uri, 'r') + self.assertRaises(NotWritableError, + lambda: fs.write_bytes(uri, bytes([0x00, 0x01]))) + + +class TestGetCachedFile(unittest.TestCase): + def setUp(self): + # Setup mock S3 bucket. + self.mock_s3 = mock_s3() + self.mock_s3.start() + self.s3 = boto3.client('s3') + self.bucket_name = 'mock_bucket' + self.s3.create_bucket(Bucket=self.bucket_name) + + self.content_str = 'hello' + self.file_name = 'hello.txt' + self.tmp_dir = rv_config.get_tmp_dir() + self.cache_dir = os.path.join(self.tmp_dir.name, 'cache') + + def tearDown(self): + self.tmp_dir.cleanup() + self.mock_s3.stop() + + def test_local(self): + local_path = os.path.join(self.tmp_dir.name, self.file_name) + str_to_file(self.content_str, local_path) + + path = get_cached_file(self.cache_dir, local_path) + self.assertTrue(os.path.isfile(path)) + + def test_local_zip(self): + local_path = os.path.join(self.tmp_dir.name, self.file_name) + local_gz_path = local_path + '.gz' + with gzip.open(local_gz_path, 'wb') as f: + f.write(bytes(self.content_str, encoding='utf-8')) + + with patch('gzip.open', side_effect=gzip.open) as patched_gzip_open: + path = get_cached_file(self.cache_dir, local_gz_path) + self.assertTrue(os.path.isfile(path)) + self.assertNotEqual(path, local_gz_path) + with open(path, 'r') as f: + self.assertEqual(f.read(), self.content_str) + + # Check that calling it again doesn't invoke the gzip.open method again. + path = get_cached_file(self.cache_dir, local_gz_path) + self.assertTrue(os.path.isfile(path)) + self.assertNotEqual(path, local_gz_path) + with open(path, 'r') as f: + self.assertEqual(f.read(), self.content_str) + self.assertEqual(patched_gzip_open.call_count, 1) + + def test_remote(self): + with patch( + 'rastervision2.pipeline.file_system.utils.download_if_needed', + side_effect=download_if_needed) as patched_download: + s3_path = 's3://{}/{}'.format(self.bucket_name, self.file_name) + str_to_file(self.content_str, s3_path) + path = get_cached_file(self.cache_dir, s3_path) + self.assertTrue(os.path.isfile(path)) + + # Check that calling it again doesn't invoke the download method again. + self.assertTrue(os.path.isfile(path)) + self.assertEqual(patched_download.call_count, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/pipeline/test_utils.py b/tests_v2/pipeline/test_utils.py new file mode 100644 index 000000000..c924cc960 --- /dev/null +++ b/tests_v2/pipeline/test_utils.py @@ -0,0 +1,24 @@ +import unittest + +from rastervision2.pipeline.utils import split_into_groups + + +class TestUtils(unittest.TestCase): + def test_split_into_groups(self): + lst = [1, 2, 3, 4, 5, 6] + + g1 = split_into_groups(lst[:5], 3) + self.assertEqual(g1, [[1, 2], [3, 4], [5]]) + + g2 = split_into_groups(lst, 7) + self.assertEqual(g2, [[1], [2], [3], [4], [5], [6]]) + + g3 = split_into_groups(lst[0:1], 7) + self.assertEqual(g3, [[1]]) + + g4 = split_into_groups(lst, 3) + self.assertEqual(g4, [[1, 2], [3, 4], [5, 6]]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests_v2/pytorch_learner/__init__.py b/tests_v2/pytorch_learner/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_v2/pytorch_learner/test_utils.py b/tests_v2/pytorch_learner/test_utils.py new file mode 100644 index 000000000..d82a01669 --- /dev/null +++ b/tests_v2/pytorch_learner/test_utils.py @@ -0,0 +1,80 @@ +import unittest + +import torch + +from rastervision2.pytorch_learner.utils import ( + compute_conf_mat, compute_conf_mat_metrics) + + +class TestComputeConfMat(unittest.TestCase): + def test1(self): + y = torch.tensor([0, 1, 0, 1]) + out = torch.tensor([0, 1, 0, 1]) + num_labels = 2 + conf_mat = compute_conf_mat(out, y, num_labels) + exp_conf_mat = torch.tensor([[2., 0], [0, 2]]) + self.assertTrue(conf_mat.equal(exp_conf_mat)) + + def test2(self): + y = torch.tensor([0, 1, 0, 1]) + out = torch.tensor([1, 1, 1, 1]) + num_labels = 2 + conf_mat = compute_conf_mat(out, y, num_labels) + exp_conf_mat = torch.tensor([[0., 2], [0, 2]]) + self.assertTrue(conf_mat.equal(exp_conf_mat)) + + +class TestComputeConfMatMetrics(unittest.TestCase): + def test1(self): + label_names = ['a', 'b'] + conf_mat = torch.tensor([[2., 0], [0, 2]]) + metrics = compute_conf_mat_metrics(conf_mat, label_names) + exp_metrics = { + 'avg_precision': 1.0, 'avg_recall': 1.0, 'avg_f1': 1.0, + 'a_precision': 1.0, 'a_recall': 1.0, 'a_f1': 1.0, + 'b_precision': 1.0, 'b_recall': 1.0, 'b_f1': 1.0} + self.assertDictEqual(metrics, exp_metrics) + + def test2(self): + label_names = ['a', 'b'] + conf_mat = torch.tensor([[0, 2.], [2, 0]]) + metrics = compute_conf_mat_metrics(conf_mat, label_names) + exp_metrics = { + 'avg_precision': 0.0, 'avg_recall': 0.0, 'avg_f1': 0.0, + 'a_precision': 0.0, 'a_recall': 0.0, 'a_f1': 0.0, + 'b_precision': 0.0, 'b_recall': 0.0, 'b_f1': 0.0} + self.assertDictEqual(metrics, exp_metrics) + + def test3(self): + label_names = ['a', 'b'] + conf_mat = torch.tensor([[1, 2], [1, 2.]]) + metrics = compute_conf_mat_metrics(conf_mat, label_names, eps=0.0) + + def f1(prec, rec): + return 2 * (prec * rec) / (prec + rec) + + def mean(a, b): + return (a + b) / 2 + + def round_dict(d): + return dict([(k, round(v, 3)) for k, v in d.items()]) + + a_prec = 1 / 2 + a_rec = 1 / 3 + a_f1 = f1(a_prec, a_rec) + b_prec = 2 / 4 + b_rec = 2 / 3 + b_f1 = f1(b_prec, b_rec) + avg_prec = mean(a_prec, b_prec) + avg_rec = mean(a_rec, b_rec) + avg_f1 = f1(avg_prec, avg_rec) + + exp_metrics = { + 'avg_precision': avg_prec, 'avg_recall': avg_rec, 'avg_f1': avg_f1, + 'a_precision': a_prec, 'a_recall': a_rec, 'a_f1': a_f1, + 'b_precision': b_prec, 'b_recall': b_rec, 'b_f1': b_f1} + self.assertDictEqual(round_dict(metrics), round_dict(exp_metrics)) + + +if __name__ == '__main__': + unittest.main()