Merge pull request #1 from HakaiInstitute/develop

Better logging, recommend new versions

las_trx/__main__.py

@@ -1,12 +1,12 @@
 import logging
 import os.path
 import queue
+import sys
 from datetime import date
 from multiprocessing import freeze_support
 from pathlib import Path
 from queue import Queue
 
-import sys
 from PyQt6 import uic
 from PyQt6.QtCore import QThread, pyqtSignal as Signal
 from PyQt6.QtGui import QIcon, QTextCursor
@@ -27,6 +27,7 @@
     sync_missing_grid_files,
     utm_zone_to_coord_type,
     resource_path,
+    get_upgrade_version
 )
 from las_trx.worker import TransformWorker
 
@@ -42,6 +43,15 @@ def __init__(self):
         self.setWindowIcon(QIcon(resource_path("resources/las-trx.ico")))
         self.setWindowTitle(f"LAS TRX v{__version__}")
 
+        upgrade_version = get_upgrade_version(__version__)
+        if upgrade_version is None:
+            self.label_upgrade_link.hide()
+        else:
+            self.label_upgrade_link.setText(
+                f"<a href=\"{upgrade_version['html_url']}\">"
+                f"New version available (v{upgrade_version['tag_name']})"
+                f"</a>")
+
         self.done_msg_box = QMessageBox(self)
         self.done_msg_box.setText("File(s) converted successfully")
         self.done_msg_box.setWindowTitle("Success")
@@ -270,13 +280,29 @@ def run(self):
     log_write_stream = LogWriteStream(log_msg_queue)
     log_handler = logging.StreamHandler(log_write_stream)
 
-    logging.basicConfig(level=logging.INFO, handlers=[log_handler])
+    log_level = logging.DEBUG if os.getenv("DEBUG") else logging.INFO
+    logging.basicConfig(level=log_level, handlers=[log_handler])
 
     app = QApplication(sys.argv)
     window = MainWindow()
+
+    if os.getenv("DEBUG"):
+        # self.lineEdit_input_file.setText(
+        #     "/mnt/aco-uvic/2020_Acquisitions/02_processed/20_3028_01_FraserRiver_ChimmneyCreek_WestWilliams_Canyon/01_LiDAR/01_Deliverables/02_QC/02_Classified_tiles/546000_5768000_5m_stp_arch_grnd_denoised.laz")
+        window.lineEdit_input_file.setText(
+            "/home/taylor/PycharmProjects/Las-TRX/testfiles/20_3028_01/*.laz")
+        window.comboBox_input_reference.setCurrentText("ITRF2014")
+        window.dateEdit_input_epoch.setDate(date(2020, 8, 12))
+
+        window.lineEdit_output_file.setText(
+            "/home/taylor/PycharmProjects/Las-TRX/testfiles/20_3028_01_converted/{}.laz")
+        window.checkBox_epoch_trans.setChecked(True)
+        window.dateEdit_output_epoch.setEnabled(True)
+        window.dateEdit_output_epoch.setDate(date(2002, 1, 1))
+        window.comboBox_output_vertical_reference.setCurrentText("CGVD2013/CGG2013a")
+
     window.show()
 
-    # When a new message is written to the log_queue via the log_write_stream, log_thread emits a signal that causes the main
     # When a new message is written to the log_queue via the log_write_stream,
     #   log_thread emits a signal that causes the main
     #   window to display that msg in the textBrowser

las_trx/resources/mainwindow.ui

@@ -757,6 +757,25 @@
      </layout>
     </widget>
    </item>
+   <item alignment="Qt::AlignRight">
+    <widget class="QLabel" name="label_upgrade_link">
+     <property name="enabled">
+      <bool>true</bool>
+     </property>
+     <property name="layoutDirection">
+      <enum>Qt::LeftToRight</enum>
+     </property>
+     <property name="text">
+      <string notr="true">&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;a href=&quot;https://github.com/HakaiInstitute/LAS-TRX/releases/tag/0.0.0&quot;&gt;&lt;span style=&quot; text-decoration: underline; color:#0000ff;&quot;&gt;New version available (v0.0.0)&lt;/span&gt;&lt;/a&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+     </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
+     <property name="textInteractionFlags">
+      <set>Qt::LinksAccessibleByKeyboard|Qt::LinksAccessibleByMouse</set>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <tabstops>

las_trx/utils.py

@@ -1,10 +1,10 @@
 import logging
+import sys
 from datetime import date
 from os import path
-from typing import TypeVar, overload
+from typing import TypeVar, overload, List, Mapping, Any, Optional
 
 import pyproj.sync
-import sys
 
 from csrspy.enums import CoordType, Reference, VerticalDatum
 
@@ -134,3 +134,48 @@ def resource_path(relative_path):
     """Get absolute path to resource, works for dev and for PyInstaller"""
     base_path = getattr(sys, "_MEIPASS", path.dirname(__file__))
     return path.abspath(path.join(base_path, relative_path))
+
+
+def _get_available_versions() -> Optional[List[Mapping[str, Any]]]:
+    import requests
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28"
+    }
+    r = requests.get("https://api.github.com/repos/HakaiInstitute/LAS-TRX/releases",
+                     headers=headers)
+    if r.status_code == requests.codes.ok:
+        return list(
+            {
+                "tag_name": version["tag_name"],
+                "html_url": version["html_url"],
+                "prerelease": version["prerelease"],
+                "draft": version["draft"],
+            }
+            for version in r.json()
+        )
+    else:
+        return None
+
+
+def get_upgrade_version(version) -> Optional[Mapping[str, str]]:
+    available_versions = _get_available_versions()
+    if available_versions is None or len(available_versions) == 0:
+        # Error fetching versions, assume no upgrade available
+        return None
+
+    # Get all tags that are newer than the current version
+    try:
+        idx = [v["tag_name"] for v in available_versions].index(version)
+    except ValueError:
+        # Current version not found in releases, so get latest version of all releases
+        idx = len(available_versions)
+
+    # Only recommend stable releases for upgrade
+    newer_stable_versions = [v for v in available_versions[:idx] if
+                             not v["prerelease"] and not v["draft"]]
+
+    if len(newer_stable_versions) == 0:
+        return None
+
+    return newer_stable_versions[0]

las_trx/worker.py

@@ -1,17 +1,17 @@
 import copy
 import logging
+import math
 import multiprocessing
 import os
 from concurrent import futures
 from pathlib import Path
+from time import sleep
 
 import laspy
-import math
 import numpy as np
 from PyQt6.QtCore import QThread, pyqtSignal as Signal
 from laspy import LasHeader
 from pyproj import CRS
-from time import sleep
 
 from csrspy import CSRSTransformer
 from las_trx.config import TransformConfig
@@ -29,9 +29,8 @@ class TransformWorker(QThread):
     success = Signal()
     error = Signal(BaseException)
 
-    def __init__(
-        self, config: TransformConfig, input_files: list[Path], output_files: list[Path]
-    ):
+    def __init__(self, config: TransformConfig, input_files: list[Path],
+                 output_files: list[Path]):
         super().__init__(parent=None)
         self.config = config
         self.input_files = input_files
@@ -43,20 +42,22 @@ def __init__(
         logger.debug(f"Will read points in chunk size of {CHUNK_SIZE}")
         logger.info("Calculating total number of iterations")
 
-        self.total_iters = 0
-        for input_file in self.input_files:
-            with laspy.open(str(input_file)) as in_las:
-                self.total_iters += math.ceil(in_las.header.point_count / CHUNK_SIZE)
-        logger.info(f"Total iterations until complete: {self.total_iters}")
-
         num_workers = min(os.cpu_count(), 61)
         self.pool = futures.ProcessPoolExecutor(max_workers=num_workers)
         self.manager = multiprocessing.Manager()
         self.lock = self.manager.RLock()
         self.current_iter = self.manager.Value("i", 0)
-
         logger.info(f"CPU process pool size: {num_workers}")
 
+        self.total_iters = 0
+        for input_file in self.input_files:
+            with laspy.open(str(input_file)) as in_las:
+                logger.debug(f"{input_file.name}: {in_las.header.point_count} points")
+                self.total_iters += math.ceil(in_las.header.point_count / CHUNK_SIZE)
+        logger.info(f"Total iterations until complete: {self.total_iters}")
+
+        self.futs = {}
+
     def check_file_names(self):
         for in_file in self.input_files:
             if in_file in self.output_files:
@@ -68,34 +69,38 @@ def check_file_names(self):
         if len(self.output_files) != len(list(set(self.output_files))):
             raise AssertionError(
                 "Duplicate output file name detected. "
-                "Use a format string for the output path to output a file based on the stem of the "
-                r"corresponding input file. e.g. 'C:\\some\path\{}_nad83csrs.laz'"
+                "Use a format string for the output path to output a file based on the "
+                "stem of the corresponding input file. "
+                r"e.g. 'C:\\some\path\{}_nad83csrs.laz'"
             )
 
     def _do_transform(self):
         self.check_file_names()
         config = self.config.dict(exclude_none=True)
-        futs = []
+        self.futs = {}
         for input_file, output_file in zip(self.input_files, self.output_files):
             if not Path(output_file).suffix:
                 output_file += ".laz"
-            logger.info(f"{input_file} -> {output_file}")
+            # logger.info(f"{input_file} -> {output_file}")
             fut = self.pool.submit(
                 transform, config, input_file, output_file, self.lock, self.current_iter
             )
             fut.add_done_callback(self.on_process_complete)
-            futs.append(fut)
+            self.futs[fut] = (input_file, output_file)
 
-        while any([f.running() for f in futs]):
+        while any([f.running() for f in self.futs.keys()]):
             self.progress.emit(self.progress_val)
             sleep(0.1)
         self.progress.emit(self.progress_val)
 
-    @staticmethod
-    def on_process_complete(fut: futures.Future):
+    def on_process_complete(self, fut: futures.Future):
+        input_file, output_file = self.futs[fut]
         err = fut.exception()
         if err is not None:
+            logger.error(f"Error transforming {input_file}")
             raise err
+        else:
+            logger.info(f"{input_file} -> {output_file}")
 
     @property
     def progress_val(self):
@@ -114,11 +119,11 @@ def run(self):
 
 
 def transform(
-    config: dict,
-    input_file: Path,
-    output_file: Path,
-    lock: multiprocessing.RLock,
-    cur: multiprocessing.Value,
+        config: dict,
+        input_file: Path,
+        output_file: Path,
+        lock: multiprocessing.RLock,
+        cur: multiprocessing.Value,
 ):
     transformer = CSRSTransformer(**config)
     config = TransformConfig(**config)
@@ -134,7 +139,7 @@ def transform(
         logger.debug(f"{laz_backend=}")
 
         with laspy.open(
-            str(output_file), mode="w", header=new_header, laz_backend=laz_backend
+                str(output_file), mode="w", header=new_header, laz_backend=laz_backend
         ) as out_las:
             for points in in_las.chunk_iterator(CHUNK_SIZE):
                 # Convert the coordinates
@@ -155,7 +160,7 @@ def transform(
 
 
 def write_header_offsets(
-    header: "LasHeader", input_file: Path, transformer: "CSRSTransformer"
+        header: "LasHeader", input_file: Path, transformer: "CSRSTransformer"
 ) -> "LasHeader":
     with laspy.open(str(input_file)) as in_las:
         points = next(in_las.chunk_iterator(CHUNK_SIZE))
@@ -174,10 +179,10 @@ def clear_header_geokeys(header: "LasHeader") -> "LasHeader":
     # Update GeoKeyDirectoryVLR
     # check and remove any existing crs vlrs
     for crs_vlr_name in (
-        "WktCoordinateSystemVlr",
-        "GeoKeyDirectoryVlr",
-        "GeoAsciiParamsVlr",
-        "GeoDoubleParamsVlr",
+            "WktCoordinateSystemVlr",
+            "GeoKeyDirectoryVlr",
+            "GeoAsciiParamsVlr",
+            "GeoDoubleParamsVlr",
     ):
         try:
             header.vlrs.extract(crs_vlr_name)