Skip to content

Commit

Permalink
optimize import and remove aliases
Browse files Browse the repository at this point in the history
  • Loading branch information
Koyaani committed Jun 9, 2022
1 parent 38aaae1 commit 6f67a79
Show file tree
Hide file tree
Showing 7 changed files with 48 additions and 43 deletions.
33 changes: 16 additions & 17 deletions py3dtiles/convert.py
Original file line number Diff line number Diff line change
@@ -1,30 +1,29 @@
import argparse
import concurrent.futures
import json
import multiprocessing
import os
import pickle
import shutil
import struct
import sys
import time
import multiprocessing
import numpy as np
import json
from collections import namedtuple
import pickle

import numpy as np
import psutil
import zmq
from pyproj import CRS, Transformer
import psutil
import struct
import concurrent.futures
import argparse

from py3dtiles import TileContentReader
from py3dtiles.constants import MIN_POINT_SIZE
from py3dtiles.utils import SrsInMissingException
from py3dtiles.points.transformations import rotation_matrix, angle_between_vectors, vector_product, inverse_matrix, scale_matrix, translation_matrix
from py3dtiles.points.utils import CommandType, ResponseType, compute_spacing, name_to_filename
from py3dtiles.points.node import Node
from py3dtiles import TileContentReader
from py3dtiles.points.shared_node_store import SharedNodeStore
import py3dtiles.points.task.las_reader as las_reader
import py3dtiles.points.task.xyz_reader as xyz_reader
import py3dtiles.points.task.node_process as node_process
import py3dtiles.points.task.pnts_writer as pnts_writer

from py3dtiles.points.task import las_reader, xyz_reader, node_process, pnts_writer
from py3dtiles.points.transformations import rotation_matrix, angle_between_vectors, vector_product, inverse_matrix, \
scale_matrix, translation_matrix
from py3dtiles.points.utils import CommandType, ResponseType, compute_spacing, name_to_filename
from py3dtiles.utils import SrsInMissingException

total_memory_MB = int(psutil.virtual_memory().total / (1024 * 1024))

Expand Down
17 changes: 9 additions & 8 deletions py3dtiles/points/node.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
import numpy as np
from pickle import dumps as pdumps, loads as ploads
import os
import json
import os
import pickle

import numpy as np

from py3dtiles import TileContentReader
from py3dtiles.constants import MIN_POINT_SIZE
from py3dtiles.feature_table import SemanticPoint
from py3dtiles.points.utils import name_to_filename, node_from_name, SubdivisionType, aabb_size_to_subdivision_type
from py3dtiles.points.points_grid import Grid
from py3dtiles.points.distance import xyz_to_child_index
from py3dtiles.points.points_grid import Grid
from py3dtiles.points.task.pnts_writer import points_to_pnts
from py3dtiles.points.utils import name_to_filename, node_from_name, SubdivisionType, aabb_size_to_subdivision_type


def node_to_tileset(args):
Expand Down Expand Up @@ -47,11 +48,11 @@ def save_to_bytes(self):
else:
sub_pickle['points'] = self.points

d = pdumps(sub_pickle)
d = pickle.dumps(sub_pickle)
return d

def load_from_bytes(self, byt):
sub_pickle = ploads(byt)
sub_pickle = pickle.loads(byt)
if 'children' in sub_pickle:
self.children = sub_pickle['children']
self.grid = sub_pickle['grid']
Expand Down Expand Up @@ -103,7 +104,7 @@ def flush_pending_points(self, catalog, scale):

def dump_pending_points(self):
result = [
(name, pdumps({'xyz': xyz, 'rgb': rgb}), len(xyz))
(name, pickle.dumps({'xyz': xyz, 'rgb': rgb}), len(xyz))
for name, xyz, rgb in self._get_pending_points()
]

Expand Down
10 changes: 6 additions & 4 deletions py3dtiles/points/node_catalog.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import math
from pickle import dumps as pdumps, loads as ploads
import pickle

import lz4.frame as gzip
from py3dtiles.points.utils import split_aabb

from py3dtiles.points.node import Node
from py3dtiles.points.utils import split_aabb


class NodeCatalog:
Expand Down Expand Up @@ -43,11 +45,11 @@ def dump(self, name, max_depth):
for n in node.children:
self.dump(n, max_depth - 1)

return pdumps(self.node_bytes)
return pickle.dumps(self.node_bytes)

def _load_from_store(self, name, data):
if len(data) > 0:
out = ploads(gzip.decompress(data))
out = pickle.loads(gzip.decompress(data))
for n in out:
spacing = self.root_spacing / math.pow(2, len(n))
aabb = self.root_aabb
Expand Down
13 changes: 7 additions & 6 deletions py3dtiles/points/task/las_reader.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import json
import numpy as np
import math
import traceback
import laspy
import pickle
import struct
import subprocess
from pickle import dumps as pdumps
import traceback

import laspy
import numpy as np

from py3dtiles.points.utils import ResponseType
from py3dtiles.utils import SrsInMissingException
Expand Down Expand Up @@ -139,11 +140,11 @@ def run(_id, filename, offset_scale, portion, queue, transformer, verbose):
[
ResponseType.NEW_TASK.value,
''.encode('ascii'),
pdumps({'xyz': coords, 'rgb': colors}),
pickle.dumps({'xyz': coords, 'rgb': colors}),
struct.pack('>I', len(coords))
], copy=False)

queue.send_multipart([ResponseType.READ.value, pdumps({'name': _id, 'total': 0})])
queue.send_multipart([ResponseType.READ.value, pickle.dumps({'name': _id, 'total': 0})])

except Exception as e:
print('Exception while reading points from las file')
Expand Down
4 changes: 2 additions & 2 deletions py3dtiles/points/task/node_process.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import os
import time
import traceback
import pickle
import struct
import time
import traceback

from py3dtiles.points.node_catalog import NodeCatalog
from py3dtiles.points.utils import ResponseType
Expand Down
8 changes: 5 additions & 3 deletions py3dtiles/points/task/pnts_writer.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import numpy as np
import os
import pickle
import struct
import os
import py3dtiles

import lz4.frame as gzip
import numpy as np

import py3dtiles
from py3dtiles.points.utils import ResponseType, name_to_filename


Expand Down
6 changes: 3 additions & 3 deletions py3dtiles/points/task/xyz_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import math
import traceback
import struct
from pickle import dumps as pdumps
import pickle

from py3dtiles.points.utils import ResponseType

Expand Down Expand Up @@ -148,13 +148,13 @@ def run(_id, filename, offset_scale, portion, queue, transformer, verbose):
[
ResponseType.NEW_TASK.value,
"".encode("ascii"),
pdumps({"xyz": coords, "rgb": colors}),
pickle.dumps({"xyz": coords, "rgb": colors}),
struct.pack(">I", len(coords)),
],
copy=False,
)

queue.send_multipart([ResponseType.READ.value, pdumps({"name": _id, "total": 0})])
queue.send_multipart([ResponseType.READ.value, pickle.dumps({"name": _id, "total": 0})])

f.close()
except Exception as e:
Expand Down

0 comments on commit 6f67a79

Please sign in to comment.