diff --git a/CHANGELOG.md b/CHANGELOG.md index 68b88d4..0d976dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## 0.4.1 - 2020-05-20 +## Added: +- select_equal filter accepts list of values to compare to the points' attributes +- also the attribute-based filter functions optionally return a mask to allow filter combinations + +## Fixed: +- bug in writing/reading 'None' as parameter in the PLY comments + ## 0.4.0 - 2020-05-13 ## Added: - build_volume module diff --git a/CITATION.cff b/CITATION.cff index 4332c06..6dc4632 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -52,7 +52,7 @@ authors: family-names: Koma given-names: Zsófia cff-version: "1.0.3" -date-released: 2020-05-13 +date-released: 2020-05-20 doi: "10.5281/zenodo.1219422" keywords: - "airborne laser scanning" @@ -62,5 +62,5 @@ keywords: license: "Apache-2.0" message: "If you use this software, please cite it using these metadata." title: "Laserchicken: toolkit for ALS point clouds" -version: "0.4.0" +version: "0.4.1" ... diff --git a/README.md b/README.md index 2a1288f..411e461 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ pip install laserchicken * Create .zenodo.json file from CITATION.cff (using cffconvert) ```cffconvert --validate``` ```cffconvert --ignore-suspect-keys --outputformat zenodo --outfile .zenodo.json``` -* Set new version number in laserchicken/_version.py +* Set new version number in laserchicken/_version.txt * Check that documentation uses the correct version * Edit Changelog (based on commits in https://github.com/eecolidar/laserchicken/compare/v0.3.2...master) * Test if package can be installed with pip (`pip install .`) diff --git a/laserchicken/_version.txt b/laserchicken/_version.txt index 1d0ba9e..267577d 100644 --- a/laserchicken/_version.txt +++ b/laserchicken/_version.txt @@ -1 +1 @@ -0.4.0 +0.4.1 diff --git a/laserchicken/filter.py b/laserchicken/filter.py index a2c23d8..21b9d1d 100644 --- a/laserchicken/filter.py +++ b/laserchicken/filter.py @@ -16,51 +16,64 @@ from laserchicken.utils import copy_point_cloud, add_metadata -def select_equal(point_cloud, attribute, value): +def select_equal(point_cloud, attribute, value, return_mask=False): """ Return the selection of the input point cloud that contains only points with a given attribute equal to some value. + If a list of values is given, select the points corresponding to any of the provided values. :param point_cloud: Input point cloud. :param attribute: The attribute name used for selection - :param value: The value to compare the attribute to - :return: A new point cloud containing only the selected points + :param value: The value(s) to compare the attribute to + :param return_mask: If true, return the mask corresponding to the selection + :return: """ _check_valid_arguments(attribute, point_cloud) - mask = point_cloud[point][attribute]['data'] == value + # broadcast using shape of the values + mask = point_cloud[point][attribute]['data'] == np.array(value)[..., None] + if mask.ndim > 1: + mask = np.any(mask, axis=0) # reduce + if return_mask: + return mask point_cloud_filtered = copy_point_cloud(point_cloud, mask) add_metadata(point_cloud_filtered, sys.modules[__name__], {'attribute': attribute, 'value': value}) return point_cloud_filtered -def select_above(point_cloud, attribute, threshold): +def select_above(point_cloud, attribute, threshold, return_mask=False): """ Return the selection of the input point cloud that contains only points with a given attribute above some value. :param point_cloud: Input point cloud :param attribute: The attribute name used for selection :param threshold: The threshold value used for selection - :return: A new point cloud containing only the selected points + :param return_mask: If true, return the mask corresponding to the selection + :return: """ _check_valid_arguments(attribute, point_cloud) mask = point_cloud[point][attribute]['data'] > threshold + if return_mask: + return mask point_cloud_filtered = copy_point_cloud(point_cloud, mask) add_metadata(point_cloud_filtered, sys.modules[__name__], {'attribute': attribute, 'threshold': threshold}) return point_cloud_filtered -def select_below(point_cloud, attribute, threshold): +def select_below(point_cloud, attribute, threshold, return_mask=False): """ Return the selection of the input point cloud that contains only points with a given attribute below some value. :param point_cloud: Input point cloud :param attribute: The attribute name used for selection :param threshold: The threshold value used for selection - :return: A new point cloud containing only the selected points + :param return_mask: If true, return the mask corresponding to the selection + :return: """ _check_valid_arguments(attribute, point_cloud) mask = point_cloud[point][attribute]['data'] < threshold + if return_mask: + return mask point_cloud_filtered = copy_point_cloud(point_cloud, mask) add_metadata(point_cloud_filtered, sys.modules[__name__], {'attribute': attribute, 'threshold': threshold}) diff --git a/laserchicken/io/ply_read.py b/laserchicken/io/ply_read.py index 55f4652..0c5de18 100644 --- a/laserchicken/io/ply_read.py +++ b/laserchicken/io/ply_read.py @@ -1,6 +1,8 @@ import ast +import json import numpy as np -from dateutil import parser + +from json.decoder import JSONDecodeError from struct import unpack, calcsize from laserchicken.io.utils import convert_to_short_type, convert_to_single_character_type @@ -78,12 +80,13 @@ def _read_header_line(ply, is_binary=False): def _read_log(comments): try: - log = ast.literal_eval(' '.join(comments)) if comments else [] - except SyntaxError: # Log can't be read. Maybe a ply file with 'regular' comments and no log. - log = [] - for i, entry in enumerate(log): - if 'time' in entry: - entry['time'] = parser.parse(entry['time']) + log = json.loads(' '.join(comments)) if comments else [] + except JSONDecodeError: + try: + # legacy: comments for laserchicken < 0.4.0 + log = ast.literal_eval(' '.join(comments)) if comments else [] + except SyntaxError: # Log can't be read. Maybe a ply file with 'regular' comments and no log. + log = [] return log diff --git a/laserchicken/io/ply_write.py b/laserchicken/io/ply_write.py index 8cc5f52..aab1a0f 100644 --- a/laserchicken/io/ply_write.py +++ b/laserchicken/io/ply_write.py @@ -99,31 +99,10 @@ def _write_comment(pc, ply): head = 'comment [\n' tail = 'comment ]\n' - formatted_entries = ',\n'.join(['comment ' + json.dumps(_stringify(entry), sort_keys=True) for entry in log]) + '\n' + formatted_entries = ',\n'.join(['comment ' + json.dumps(entry, sort_keys=True) for entry in log]) + '\n' ply.write(head + formatted_entries + tail) -def _stringify(entry): - copy = {} - for key, value in _sort_by_key(entry): - if isinstance(value, dict): - copy[key] = _stringify(value) - elif isinstance(value, list): - copy[key] = [_stringify(entry) if isinstance(entry, dict) else entry for entry in value] - else: - if key == 'time': - copy[key] = str(value) - else: - copy[key] = value - return copy - - -def _sort_by_key(entry): - key_value_pairs = list(entry.items()) - key_value_pairs.sort(key=lambda key_value_pair: key_value_pair[0]) - return key_value_pairs - - def _write_header_elements(pc, attributes, ply, element_name, get_num_elements=None): if element_name in pc: num_elements = get_num_elements(pc[element_name]) if get_num_elements else 1 diff --git a/laserchicken/io/test_read_ply.py b/laserchicken/io/test_read_ply.py index c523df5..73f4977 100644 --- a/laserchicken/io/test_read_ply.py +++ b/laserchicken/io/test_read_ply.py @@ -1,3 +1,4 @@ +import dateutil import os import shutil import unittest @@ -89,8 +90,10 @@ def test_correctModulesLogged(self): def test_correctTimesLogged(self): log = load(self.test_file_path)['log'] - self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1], list(log[0]['time'].timetuple())) - self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1], list(log[1]['time'].timetuple())) + self.assertListEqual([2018, 1, 18, 16, 1, 0, 3, 18, -1], + list(dateutil.parser.parse(log[0]['time']).timetuple())) + self.assertListEqual([2018, 1, 18, 16, 3, 0, 3, 18, -1], + list(dateutil.parser.parse(log[1]['time']).timetuple())) def setUp(self): os.mkdir(self._test_dir) diff --git a/laserchicken/test_filter.py b/laserchicken/test_filter.py index df75780..de6fea6 100644 --- a/laserchicken/test_filter.py +++ b/laserchicken/test_filter.py @@ -48,6 +48,27 @@ def test_selectEqual_outputCorrect(): pc_out = select_equal(pc_in, 'return', 1) assert_equal(len(pc_out[point]['x']['data']), 2) + @staticmethod + def test_selectEqual_multipleValues(): + """ Correct number of results. """ + pc_in = get_test_data() + pc_out = select_equal(pc_in, 'return', [1, 2]) + assert_equal(len(pc_out[point]['x']['data']), 3) + + @staticmethod + def test_selectEqual_maskCorrect(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_equal(pc_in, 'return', 1, return_mask=True) + assert_equal(mask_out, np.array([1,1,0], dtype=bool)) + + @staticmethod + def test_selectEqual_maskEmpty(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_equal(pc_in, 'return', 3, return_mask=True) + assert_equal(sum(mask_out), 0) + class TestSelectBelow(unittest.TestCase): @staticmethod @@ -88,6 +109,20 @@ def test_selectBelow_onlyOnePoint(): assert_almost_equal(pc_out[point]['y']['data'][0], 2.1) assert_almost_equal(pc_out[point]['z']['data'][0], 3.1) assert_almost_equal(pc_out[point]['return']['data'][0], 1) + + @staticmethod + def test_selectBelow_maskCorrect(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_below(pc_in, 'return', 2, return_mask=True) + assert_equal(mask_out, np.array([1,1,0], dtype=bool)) + + @staticmethod + def test_selectBelow_maskAll(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_below(pc_in, 'return', 3, return_mask=True) + assert_equal(sum(mask_out), 3) class TestSelectAbove(unittest.TestCase): @@ -128,6 +163,20 @@ def test_selectBelow_onlyOnePoint(): assert_almost_equal(pc_out[point]['y']['data'][0], 2.3) assert_almost_equal(pc_out[point]['z']['data'][0], 3.3) assert_almost_equal(pc_out[point]['return']['data'][0], 2) + + @staticmethod + def test_selectAbove_maskCorrect(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_above(pc_in, 'return', 1, return_mask=True) + assert_equal(mask_out, np.array([0,0,1], dtype=bool)) + + @staticmethod + def test_selectAbove_maskAll(): + """ Correct number of results. """ + pc_in = get_test_data() + mask_out = select_above(pc_in, 'return', 0, return_mask=True) + assert_equal(sum(mask_out), 3) class TestSelectPolygonWKT(unittest.TestCase): diff --git a/laserchicken/utils.py b/laserchicken/utils.py index 289bf1c..de538c5 100644 --- a/laserchicken/utils.py +++ b/laserchicken/utils.py @@ -140,7 +140,7 @@ def add_metadata(point_cloud, module, params): :param params: :return: """ - msg = {"time": datetime.datetime.utcnow(), + msg = {"time": str(datetime.datetime.utcnow()), "module": module.__name__ if hasattr(module, "__name__") else str(module)} if any(params): msg["parameters"] = params diff --git a/tutorial.ipynb b/tutorial.ipynb index 24ced33..5fcd126 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -44,15 +44,15 @@ " -0.24100002, -0.24000002])},\n", " 'raw_classification': {'type': 'uint8',\n", " 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n", + " 'intensity': {'type': 'uint16',\n", + " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n", " 'gps_time': {'type': 'float64',\n", " 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n", - " 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n", - " 'intensity': {'type': 'uint16',\n", - " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)}},\n", - " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n", + " 78563778.28828931, 78563778.3107884 , 78563778.32578015])}},\n", + " 'log': [{'time': '2020-05-19 14:56:55.083339',\n", " 'module': 'laserchicken.io.load',\n", " 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n", - " 'version': '0.3.2'}]}" + " 'version': '0.4.0'}]}" ] }, "execution_count": 2, @@ -91,21 +91,21 @@ " -0.24100002, -0.24000002])},\n", " 'raw_classification': {'type': 'uint8',\n", " 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n", + " 'intensity': {'type': 'uint16',\n", + " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n", " 'gps_time': {'type': 'float64',\n", " 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n", " 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n", - " 'intensity': {'type': 'uint16',\n", - " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n", " 'normalized_height': {'type': 'float64',\n", " 'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n", - " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n", + " 'log': [{'time': '2020-05-19 14:56:55.083339',\n", " 'module': 'laserchicken.io.load',\n", " 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n", - " 'version': '0.3.2'},\n", - " {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n", + " 'version': '0.4.0'},\n", + " {'time': '2020-05-19 14:56:55.149038',\n", " 'module': 'laserchicken.normalize',\n", " 'parameters': {'cell_size': None},\n", - " 'version': '0.3.2'}]}" + " 'version': '0.4.0'}]}" ] }, "execution_count": 3, @@ -137,21 +137,21 @@ " -0.24100002, -0.24000002])},\n", " 'raw_classification': {'type': 'uint8',\n", " 'data': array([9, 9, 9, ..., 9, 9, 9], dtype=uint8)},\n", + " 'intensity': {'type': 'uint16',\n", + " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n", " 'gps_time': {'type': 'float64',\n", " 'data': array([78563787.97322202, 78563787.93570042, 78563787.93571067, ...,\n", " 78563778.28828931, 78563778.3107884 , 78563778.32578015])},\n", - " 'intensity': {'type': 'uint16',\n", - " 'data': array([ 41, 152, 12, ..., 10, 15, 10], dtype=uint16)},\n", " 'normalized_height': {'type': 'float64',\n", " 'data': array([1.236, 1.311, 1.317, ..., 1.336, 1.336, 1.337])}},\n", - " 'log': [{'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 404699),\n", + " 'log': [{'time': '2020-05-19 14:56:55.083339',\n", " 'module': 'laserchicken.io.load',\n", " 'parameters': {'path': 'testdata/AHN3.las', 'args': ()},\n", - " 'version': '0.3.2'},\n", - " {'time': datetime.datetime(2020, 3, 10, 8, 1, 8, 473466),\n", + " 'version': '0.4.0'},\n", + " {'time': '2020-05-19 14:56:55.149038',\n", " 'module': 'laserchicken.normalize',\n", " 'parameters': {'cell_size': None},\n", - " 'version': '0.3.2'}]}" + " 'version': '0.4.0'}]}" ] }, "execution_count": 4, @@ -245,13 +245,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Cylinder size in Bytes: 1214225560.6124551\n", + "Cylinder size in Bytes: 1262794583.0369534\n", "Memory size in Bytes: 17179869184\n", "Start tree creation\n", "Done with env tree creation\n", "Done with target tree creation\n", - "Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.24 seconds\n", - "Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.18 seconds\n", + "Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\"Extracting feature(s) \"['eigenv_1', 'eigenv_2', 'eigenv_3', 'normal_vector_1', 'normal_vector_2', 'normal_vector_3', 'slope']\" took 0.22 seconds\n", + "Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\"Extracting feature(s) \"['mean_z', 'std_z', 'coeff_var_z']\" took 0.21 seconds\n", "The following unrequested features were calculated as a side effect, but will not be returned: ['normal_vector_3', 'normal_vector_2', 'normal_vector_1', 'eigenv_3', 'eigenv_2', 'eigenv_1', 'coeff_var_z']\n" ] } @@ -574,12 +574,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Cylinder size in Bytes: 1214225560.6124551\n", + "Cylinder size in Bytes: 1262794583.0369534\n", "Memory size in Bytes: 17179869184\n", "Start tree creation\n", "Done with env tree creation\n", "Done with target tree creation\n", - "Extracting feature(s) \"['band_ratio_1