Merge branch 'point-format-class'

docs/api/index.rst

@@ -22,4 +22,5 @@ Submodules
    pylas.errors
    pylas.compression
    pylas.headers
+   pylas.point.format
 

docs/api/pylas.point.format.rst

@@ -0,0 +1,7 @@
+pylas.point.format module
+=========================
+
+.. automodule:: pylas.point.format
+    :members:
+    :undoc-members:
+    :show-inheritance:

pylas/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.2.0'
+__version__ = "0.2.0"
 
 from . import errors, vlrs
 from .headers import HeaderFactory
@@ -7,5 +7,5 @@
 from .lib import mmap_las as mmap
 from .lib import open_las as open
 from .lib import read_las as read
-from .point.dims import supported_point_formats, lost_dimensions
-
+from .point import PointFormat
+from .point.dims import supported_point_formats

pylas/compression.py

@@ -10,7 +10,6 @@
 import numpy as np
 
 from .errors import LazPerfNotFound, PylasError
-from .point.dims import get_dtype_of_format_id
 
 HAS_LAZPERF = False
 
@@ -27,8 +26,10 @@
 def raise_if_no_lazperf():
     if not HAS_LAZPERF:
         raise LazPerfNotFound("Cannot manipulate laz data")
-    elif lazperf.__version__ < '1.3.0':
-        raise LazPerfNotFound("Version >= 1.3.0 required, you have {}".format(lazperf.__version__))
+    elif lazperf.__version__ < "1.3.0":
+        raise LazPerfNotFound(
+            "Version >= 1.3.0 required, you have {}".format(lazperf.__version__)
+        )
 
 
 def is_point_format_compressed(point_format_id):
@@ -53,7 +54,9 @@ def decompress_buffer(compressed_buffer, points_dtype, point_count, laszip_vlr):
     point_compressed = np.frombuffer(compressed_buffer, dtype=np.uint8)
 
     vlr_data = np.frombuffer(laszip_vlr.record_data, dtype=np.uint8)
-    decompressor = lazperf.VLRDecompressor(point_compressed, points_dtype.itemsize, vlr_data)
+    decompressor = lazperf.VLRDecompressor(
+        point_compressed, points_dtype.itemsize, vlr_data
+    )
 
     point_uncompressed = decompressor.decompress_points(point_count)
 
@@ -68,7 +71,7 @@ def create_laz_vlr(points_record):
     raise_if_no_lazperf()
     record_schema = lazperf.RecordSchema()
 
-    if points_record.point_format_id >= 6:
+    if points_record.point_format.id >= 6:
         raise PylasError("Can't compress points with format it >= 6")
     record_schema.add_point()
 
@@ -78,13 +81,13 @@ def create_laz_vlr(points_record):
     if "red" in points_record.dimensions_names:
         record_schema.add_rgb()
 
-    official_dtype = get_dtype_of_format_id(points_record.point_format_id)
-
-    num_extra_bytes = points_record.array.dtype.itemsize - official_dtype.itemsize
+    num_extra_bytes = points_record.point_format.num_extra_bytes
     if num_extra_bytes > 0:
         record_schema.add_extra_bytes(num_extra_bytes)
     elif num_extra_bytes < 0:
-        raise PylasError("Incoherent number of extra bytes ({})".format(num_extra_bytes))
+        raise PylasError(
+            "Incoherent number of extra bytes ({})".format(num_extra_bytes)
+        )
 
     return lazperf.LazVLR(record_schema)
 

pylas/extradims.py

@@ -1,6 +1,18 @@
 from . import errors
 
-_extra_dims_base_style_1 = ("", "u1", "i1", "u2", "i2", "u4", "i4", "u8", "i8", "f4", "f8")
+_extra_dims_base_style_1 = (
+    "",
+    "u1",
+    "i1",
+    "u2",
+    "i2",
+    "u4",
+    "i4",
+    "u8",
+    "i8",
+    "f4",
+    "f8",
+)
 _extra_dims_base_style_2 = (
     "",
     "uint8",
@@ -15,17 +27,33 @@
     "double",
 )
 
-_extra_dims_style_1_array_2 = tuple("2{}".format(_type) for _type in _extra_dims_base_style_1[1:])
-_extra_dims_style_1_array_3 = tuple("3{}".format(_type) for _type in _extra_dims_base_style_1[1:])
+_extra_dims_style_1_array_2 = tuple(
+    "2{}".format(_type) for _type in _extra_dims_base_style_1[1:]
+)
+_extra_dims_style_1_array_3 = tuple(
+    "3{}".format(_type) for _type in _extra_dims_base_style_1[1:]
+)
 
-_extra_dims_style_2_array_2 = tuple("2{}".format(_type) for _type in _extra_dims_base_style_2[1:])
-_extra_dims_style_2_array_3 = tuple("3{}".format(_type) for _type in _extra_dims_base_style_2[1:])
+_extra_dims_style_2_array_2 = tuple(
+    "2{}".format(_type) for _type in _extra_dims_base_style_2[1:]
+)
+_extra_dims_style_2_array_3 = tuple(
+    "3{}".format(_type) for _type in _extra_dims_base_style_2[1:]
+)
 
-_extra_dims_style_1 = _extra_dims_base_style_1 + _extra_dims_style_1_array_2 + _extra_dims_style_1_array_3
-_extra_dims_style_2 = _extra_dims_base_style_2 + _extra_dims_style_1_array_2 + _extra_dims_style_2_array_3
+_extra_dims_style_1 = (
+    _extra_dims_base_style_1 + _extra_dims_style_1_array_2 + _extra_dims_style_1_array_3
+)
+_extra_dims_style_2 = (
+    _extra_dims_base_style_2 + _extra_dims_style_1_array_2 + _extra_dims_style_2_array_3
+)
 
-_type_to_extra_dim_id_style_1 = {type_str: i for i, type_str in enumerate(_extra_dims_style_1)}
-_type_to_extra_dim_id_style_2 = {type_str: i for i, type_str in enumerate(_extra_dims_style_2)}
+_type_to_extra_dim_id_style_1 = {
+    type_str: i for i, type_str in enumerate(_extra_dims_style_1)
+}
+_type_to_extra_dim_id_style_2 = {
+    type_str: i for i, type_str in enumerate(_extra_dims_style_2)
+}
 
 
 def get_type_for_extra_dim(type_index):

pylas/headers/rawheader.py

@@ -286,6 +286,7 @@ class HeaderFactory:
     This Factory also handles converting headers between different
     versions.
     """
+
     _version_to_header = {
         "1.1": RawHeader1_1,
         "1.2": RawHeader1_2,

pylas/lasdatas/base.py

@@ -1,8 +1,9 @@
-import numpy as np
 import logging
 
+import numpy as np
+
 from ..compression import compress_buffer, create_laz_vlr, uncompressed_id_to_compressed
-from ..point import record, dims
+from ..point import record, dims, PointFormat
 from ..vlrs import known, vlrlist
 
 logger = logging.getLogger(__name__)
@@ -40,7 +41,7 @@ class LasBase(object):
 
     def __init__(self, *, header, vlrs=None, points=None):
         if points is None:
-            points = record.PackedPointRecord.empty(header.point_format_id)
+            points = record.PackedPointRecord.empty(PointFormat(header.point_format_id))
         self.__dict__["points_data"] = points
         self.header = header
         self.vlrs = vlrs if vlrs is not None else vlrlist.VLRList()
@@ -102,7 +103,7 @@ def points(self, value):
         """
         new_point_record = record.PackedPointRecord(value)
         dims.raise_if_version_not_compatible_with_fmt(
-            new_point_record.point_format_id, self.header.version
+            new_point_record.point_format.id, self.header.version
         )
         self.points_data = new_point_record
         self.update_header()
@@ -148,7 +149,7 @@ def __setitem__(self, key, value):
         self.points_data[key] = value
 
     def update_header(self):
-        self.header.point_format_id = self.points_data.point_format_id
+        self.header.point_format_id = self.points_data.point_format
         self.header.point_count = len(self.points_data)
         self.header.point_data_record_length = self.points_data.point_size
 
@@ -177,10 +178,15 @@ def write_to(self, out_stream, do_compress=False):
 
         self.update_header()
 
-        if self.vlrs.get('ExtraBytesVlr') and not self.points_data.extra_dimensions_names:
-            logger.error("Las contains an ExtraBytesVlr, but no extra bytes were found in the point_record, "
-                         "removing the vlr")
-            self.vlrs.extract('ExtraBytesVlr')
+        if (
+                self.vlrs.get("ExtraBytesVlr")
+                and not self.points_data.extra_dimensions_names
+        ):
+            logger.error(
+                "Las contains an ExtraBytesVlr, but no extra bytes were found in the point_record, "
+                "removing the vlr"
+            )
+            self.vlrs.extract("ExtraBytesVlr")
 
         if do_compress:
             laz_vrl = create_laz_vlr(self.points_data)
@@ -283,7 +289,7 @@ def __repr__(self):
         return "<LasData({}.{}, point fmt: {}, {} points, {} vlrs)>".format(
             self.header.version_major,
             self.header.version_minor,
-            self.points_data.point_format_id,
+            self.points_data.point_format,
             len(self.points_data),
             len(self.vlrs),
         )

pylas/lasdatas/las14.py

@@ -32,7 +32,7 @@ def add_extra_dim(self, name, type, description=""):
             self.points_data.add_extra_dims([(name, type)])
 
     def write_to(self, out_stream, do_compress=False):
-        if do_compress and self.points_data.point_format_id >= 6:
+        if do_compress and self.points_data.point_format >= 6:
             raise NotImplementedError(
                 "LazPerf cannot compress 1.4 files with point format >= 6"
             )

pylas/lasmmap.py

@@ -1,12 +1,11 @@
 import mmap
 
 from . import headers
+from . import lasreader
 from .lasdatas import base
-from .point import record
+from .point import PointFormat, record
 from .vlrs import vlrlist
 
-from . import lasreader
-
 WHOLE_FILE = 0
 
 
@@ -32,6 +31,7 @@ def __init__(self, filename):
         m = mmap.mmap(fileref.fileno(), length=WHOLE_FILE, access=mmap.ACCESS_WRITE)
         header = headers.HeaderFactory.from_mmap(m)
         if header.are_points_compressed:
+            m.close()
             raise ValueError("Cannot mmap a compressed LAZ file")
         super().__init__(header=header)
         self.fileref, self.mmap = fileref, m
@@ -43,12 +43,12 @@ def __init__(self, filename):
         except IndexError:
             extra_dims = None
 
+        point_format = PointFormat(self.header.point_format_id, extra_dims=extra_dims)
         self.points_data = record.PackedPointRecord.from_buffer(
             self.mmap,
             self.header.point_format_id,
             count=self.header.point_count,
             offset=self.header.offset_to_point_data,
-            extra_dims=extra_dims,
         )
 
     def _write_vlrs(self):

pylas/lasreader.py

@@ -5,7 +5,7 @@
 from . import headers, errors, evlrs
 from .compression import laszip_decompress
 from .lasdatas import las14, las12
-from .point import dims, record
+from .point import record, PointFormat
 from .vlrs import rawvlr
 from .vlrs.vlrlist import VLRList
 
@@ -66,7 +66,7 @@ def read(self):
             self.__init__(io.BytesIO(laszip_decompress(self.stream)))
             return self.read()
 
-        if dims.format_has_waveform_packet(self.header.point_format_id):
+        if points.point_format.has_waveform_packet:
             self.stream.seek(
                 self.start_pos + self.header.start_of_waveform_data_packet_record
             )
@@ -106,19 +106,17 @@ def _read_points(self, vlrs):
         except IndexError:
             extra_dims = None
 
+        point_format = PointFormat(self.header.point_format_id, extra_dims=extra_dims)
         if self.header.are_points_compressed:
             laszip_vlr = vlrs.pop(vlrs.index("LasZipVlr"))
-            points = self._read_compressed_points_data(laszip_vlr, extra_dims)
+            points = self._read_compressed_points_data(laszip_vlr, point_format)
         else:
             points = record.PackedPointRecord.from_stream(
-                self.stream,
-                self.header.point_format_id,
-                self.header.point_count,
-                extra_dims,
+                self.stream, point_format, self.header.point_count
             )
         return points
 
-    def _read_compressed_points_data(self, laszip_vlr, extra_dims):
+    def _read_compressed_points_data(self, laszip_vlr, point_format):
         """ reads the compressed point record
         """
         offset_to_chunk_table = struct.unpack("<q", self.stream.read(8))[0]
@@ -134,10 +132,9 @@ def _read_compressed_points_data(self, laszip_vlr, extra_dims):
 
         points = record.PackedPointRecord.from_compressed_buffer(
             self.stream.read(size_of_point_data),
-            self.header.point_format_id,
+            point_format,
             self.header.point_count,
             laszip_vlr,
-            extra_dims=extra_dims
         )
         return points
 

pylas/lib.py

@@ -10,7 +10,7 @@
 from .lasdatas import las12, las14
 from .lasmmap import LasMMAP
 from .lasreader import LasReader
-from .point import dims, record
+from .point import dims, record, PointFormat
 
 USE_UNPACKED = False
 
@@ -114,7 +114,7 @@ def create_from_header(header):
     pylas.lasdatas.base.LasBase
     """
     header = copy.copy(header)
-    points = record.PackedPointRecord.zeros(header.point_format_id, header.point_count)
+    points = record.PackedPointRecord.zeros(PointFormat(header.point_format_id), header.point_count)
     if header.version >= "1.4":
         return las14.LasData(header=header, points=points)
     return las12.LasData(header=header, points=points)
@@ -232,7 +232,7 @@ def convert(source_las, *, point_format_id=None, file_version=None):
         pylas.lasdatas.base.LasBase
     """
     if point_format_id is None:
-        point_format_id = source_las.points_data.point_format_id
+        point_format_id = source_las.points_data.point_format
 
     if file_version is None:
         file_version = max(
@@ -246,20 +246,25 @@ def convert(source_las, *, point_format_id=None, file_version=None):
     header = headers.HeaderFactory.convert_header(source_las.header, file_version)
     header.point_format_id = point_format_id
 
+    point_format = PointFormat(point_format_id, source_las.points_data.point_format.extra_dims)
     points = record.PackedPointRecord.from_point_record(
-        source_las.points_data, point_format_id
+        source_las.points_data, point_format
     )
 
-    try:
-        evlrs = source_las.evlrs
-    except ValueError:
-        evlrs = []
-
     if file_version >= "1.4":
-        return las14.LasData(
+        try:
+            evlrs = source_las.evlrs
+        except ValueError:
+            evlrs = []
+
+        las = las14.LasData(
             header=header, vlrs=source_las.vlrs, points=points, evlrs=evlrs
         )
-    return las12.LasData(header=header, vlrs=source_las.vlrs, points=points)
+    else:
+        las = las12.LasData(header=header, vlrs=source_las.vlrs, points=points)
+
+    las.vlrs = copy.deepcopy(source_las.vlrs)
+    return las
 
 
 def merge_las(*las_files):
@@ -285,7 +290,7 @@ def merge_las(*las_files):
         raise ValueError("No files to merge")
 
     if not utils.files_have_same_dtype(las_files):
-        raise ValueError('All files must have the same point format')
+        raise ValueError("All files must have the same point format")
 
     header = las_files[0].header
     num_pts_merged = sum(len(las.points) for las in las_files)

pylas/point/__init__.py

@@ -0,0 +1 @@
+from .format import PointFormat