Skip to content
This repository has been archived by the owner on Aug 18, 2022. It is now read-only.

Commit

Permalink
Use readinto to avoid copying from bytes to bytearray
Browse files Browse the repository at this point in the history
Less copy = faster reading + less ram used
  • Loading branch information
tmontaigu committed Oct 13, 2020
1 parent c403716 commit 9bf7496
Showing 1 changed file with 15 additions and 9 deletions.
24 changes: 15 additions & 9 deletions pylas/lasreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def read_n_points(self, n: int) -> Optional[record.ScaleAwarePointRecord]:
n = min(n, points_left)

r = record.PackedPointRecord.from_buffer(
bytearray(self.point_source.read_n_points(n)), self.point_format, n
self.point_source.read_n_points(n), self.point_format, n
)
points = record.ScaleAwarePointRecord(
r.array, r.point_format, self.header.scales, self.header.offsets
Expand Down Expand Up @@ -219,7 +219,7 @@ class IPointReader(abc.ABC):
"""

@abc.abstractmethod
def read_n_points(self, n) -> bytes:
def read_n_points(self, n) -> bytearray:
...

@abc.abstractmethod
Expand All @@ -234,8 +234,13 @@ def __init__(self, source, point_size) -> None:
self.source = source
self.point_size = point_size

def read_n_points(self, n) -> bytes:
return self.source.read(n * self.point_size)
def read_n_points(self, n) -> bytearray:
try:
data = bytearray(n * self.point_size)
self.source.readinto(data)
return data
except AttributeError:
return bytearray(self.source.read(n * self.point_size))

def close(self):
self.source.close()
Expand Down Expand Up @@ -288,11 +293,12 @@ def __init__(self, source, header, _vlrs) -> None:
stderr=subprocess.PIPE,
)

def read_n_points(self, n) -> bytes:
b = self.process.stdout.read(n * self.point_size)
def read_n_points(self, n) -> bytearray:
b = bytearray(n * self.point_size)
self.process.stdout.readinto(b)
if self.process.poll() is not None:
self.raise_if_bad_err_code()
return b
return bytearray(b)

def close(self):
if self.conveyor is not None and self.conveyor.is_alive():
Expand Down Expand Up @@ -337,8 +343,8 @@ def __init__(self, source, laszip_vlr: LasZipVlr, parallel: bool) -> None:
else:
self.decompressor = lazrs.LasZipDecompressor(source, laszip_vlr.record_data)

def read_n_points(self, n) -> bytes:
point_bytes = np.zeros(n * self.vlr.item_size(), np.uint8)
def read_n_points(self, n) -> bytearray:
point_bytes = bytearray(n * self.vlr.item_size())
self.decompressor.decompress_many(point_bytes)
return point_bytes

Expand Down

0 comments on commit 9bf7496

Please sign in to comment.