Concurrency refactor 999 (#1029)

* Working on making export threadsafe, #999

* Working on making export threadsafe, #999

* refactor for concurrent export, #999

* Fixed race condition in ExportRecord context manager

osxphotos/_constants.py

@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+import logging
 import os.path
+import sqlite3
 from datetime import datetime
 from enum import Enum
 
+logger: logging.Logger = logging.getLogger("osxphotos")
+
 APP_NAME = "osxphotos"
 
 OSXPHOTOS_URL = "https://github.com/RhetTbull/osxphotos"
@@ -464,3 +468,12 @@ class AlbumSortOrder(Enum):
 UUID_PATTERN = (
     r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
 )
+# Reference: https://docs.python.org/3/library/sqlite3.html?highlight=sqlite3%20threadsafety#sqlite3.threadsafety
+# and https://docs.python.org/3/library/sqlite3.html?highlight=sqlite3%20threadsafety#sqlite3.connect
+# 3: serialized mode; Threads may share the module, connections and cursors
+# 3 is the default in the python.org python 3.11 distribution
+# earlier versions of python.org python 3.x default to 1 which means threads may not share
+# sqlite3 connections and thus PhotoInfo.export() cannot be used in a multithreaded environment
+# pass SQLITE_CHECK_SAME_THREAD to sqlite3.connect() to enable multithreaded access on systems that support it
+SQLITE_CHECK_SAME_THREAD = not sqlite3.threadsafety == 3
+logger.debug(f"{SQLITE_CHECK_SAME_THREAD=}, {sqlite3.threadsafety=}")

osxphotos/albuminfo.py

@@ -18,6 +18,7 @@
     _PHOTOS_4_VERSION,
     _PHOTOS_5_ALBUM_KIND,
     _PHOTOS_5_FOLDER_KIND,
+    _PHOTOS_5_VERSION,
     TIME_DELTA,
     AlbumSortOrder,
 )
@@ -61,7 +62,7 @@ class AlbumInfoBaseClass:
     including folders, photos, etc.
     """
 
-    def __init__(self, db=None, uuid=None):
+    def __init__(self, db, uuid):
         self._uuid = uuid
         self._db = db
         self._title = self._db._dbalbum_details[uuid]["title"]
@@ -121,7 +122,8 @@ def start_date(self):
     @property
     def end_date(self):
         """For Albums, return end date (most recent image) of album or None for albums with no images
-        For Import Sessions, return end date of import sessions (when import was completed)"""
+        For Import Sessions, return end date of import sessions (when import was completed)
+        """
         try:
             return self._end_date
         except AttributeError:
@@ -163,6 +165,17 @@ def owner(self):
                 self._owner = None
             return self._owner
 
+    def asdict(self):
+        """Return album info as a dict"""
+        return {
+            "uuid": self.uuid,
+            "creation_date": self.creation_date,
+            "start_date": self.start_date,
+            "end_date": self.end_date,
+            "owner": self.owner,
+            "photos": [p.uuid for p in self.photos],
+        }
+
     def __len__(self):
         """return number of photos contained in album"""
         return len(self.photos)
@@ -174,6 +187,10 @@ class AlbumInfo(AlbumInfoBaseClass):
     including folders, photos, etc.
     """
 
+    def __init__(self, db, uuid):
+        super().__init__(db=db, uuid=uuid)
+        self._title = self._db._dbalbum_details[uuid]["title"]
+
     @property
     def title(self):
         """return title / name of album"""
@@ -205,10 +222,11 @@ def photos(self):
 
     @property
     def folder_names(self):
-        """return hierarchical list of folders the album is contained in
+        """Return hierarchical list of folders the album is contained in
         the folder list is in form:
         ["Top level folder", "sub folder 1", "sub folder 2", ...]
-        returns empty list if album is not in any folders"""
+        or empty list if album is not in any folders
+        """
 
         try:
             return self._folder_names
@@ -218,10 +236,9 @@ def folder_names(self):
 
     @property
     def folder_list(self):
-        """return hierarchical list of folders the album is contained in
-        as list of FolderInfo objects in form
-        ["Top level folder", "sub folder 1", "sub folder 2", ...]
-        returns empty list if album is not in any folders"""
+        """Returns list of FolderInfo objects for each folder the album is contained in
+        or empty list if album is not in any folders
+        """
 
         try:
             return self._folders
@@ -246,7 +263,7 @@ def parent(self):
                 parent_pk = self._db._dbalbum_details[self._uuid]["parentfolder"]
                 self._parent = (
                     FolderInfo(db=self._db, uuid=self._db._dbalbums_pk[parent_pk])
-                    if parent_pk != self._db._folder_root_pk
+                    if parent_pk is not None and parent_pk != self._db._folder_root_pk
                     else None
                 )
             return self._parent
@@ -281,34 +298,88 @@ def photo_index(self, photo):
             f"Photo with uuid {photo.uuid} does not appear to be in this album"
         )
 
+    def asdict(self):
+        """Return album info as a dict"""
+        dict_data = super().asdict()
+        dict_data["title"] = self.title
+        dict_data["folder_names"] = self.folder_names
+        dict_data["folder_list"] = [f.uuid for f in self.folder_list]
+        dict_data["sort_order"] = self.sort_order
+        dict_data["parent"] = self.parent.uuid if self.parent else None
+        return dict_data
+
 
 class ImportInfo(AlbumInfoBaseClass):
     """Information about import sessions"""
 
+    def __init__(self, db, uuid):
+        self._uuid = uuid
+        self._db = db
+
+        if self._db._db_version >= _PHOTOS_5_VERSION:
+            return super().__init__(db=db, uuid=uuid)
+
+        import_session = self._db._db_import_group[self._uuid]
+        try:
+            self._creation_date_timestamp = import_session[3]
+        except (ValueError, TypeError, KeyError):
+            self._creation_date_timestamp = datetime(1970, 1, 1)
+        self._start_date_timestamp = self._creation_date_timestamp
+        self._end_date_timestamp = self._creation_date_timestamp
+        self._title = import_session[2]
+        self._local_tz = get_local_tz(
+            datetime.fromtimestamp(self._creation_date_timestamp + TIME_DELTA)
+        )
+
+    @property
+    def title(self):
+        """return title / name of import session"""
+        return self._title
+
     @property
     def photos(self):
         """return list of photos contained in import session"""
         try:
             return self._photos
         except AttributeError:
-            uuid_list, sort_order = zip(
-                *[
-                    (uuid, self._db._dbphotos[uuid]["fok_import_session"])
-                    for uuid in self._db._dbphotos
-                    if self._db._dbphotos[uuid]["import_uuid"] == self.uuid
+            if self._db._db_version >= _PHOTOS_5_VERSION:
+                uuid_list, sort_order = zip(
+                    *[
+                        (uuid, self._db._dbphotos[uuid]["fok_import_session"])
+                        for uuid in self._db._dbphotos
+                        if self._db._dbphotos[uuid]["import_uuid"] == self.uuid
+                    ]
+                )
+                sorted_uuid = sort_list_by_keys(uuid_list, sort_order)
+                self._photos = self._db.photos_by_uuid(sorted_uuid)
+            else:
+                import_photo_uuids = [
+                    u
+                    for u in self._db._dbphotos
+                    if self._db._dbphotos[u]["import_uuid"] == self.uuid
                 ]
-            )
-            sorted_uuid = sort_list_by_keys(uuid_list, sort_order)
-            self._photos = self._db.photos_by_uuid(sorted_uuid)
+                self._photos = self._db.photos_by_uuid(import_photo_uuids)
             return self._photos
 
+    def asdict(self):
+        """Return import info as a dict"""
+        return {
+            "uuid": self.uuid,
+            "creation_date": self.creation_date,
+            "start_date": self.start_date,
+            "end_date": self.end_date,
+            "title": self.title,
+            "photos": [p.uuid for p in self.photos],
+        }
+
     def __bool__(self):
         """Always returns True
         A photo without an import session will return None for import_info,
         thus if import_info is not None, it must be a valid import_info object (#820)
         """
         return True
 
+
 class ProjectInfo(AlbumInfo):
     """
     ProjectInfo with info about projects
@@ -386,7 +457,7 @@ def parent(self):
                 parent_pk = self._db._dbalbum_details[self._uuid]["parentfolder"]
                 self._parent = (
                     FolderInfo(db=self._db, uuid=self._db._dbalbums_pk[parent_pk])
-                    if parent_pk != self._db._folder_root_pk
+                    if parent_pk is not None and parent_pk != self._db._folder_root_pk
                     else None
                 )
             return self._parent
@@ -416,6 +487,16 @@ def subfolders(self):
             self._folders = folders
             return self._folders
 
+    def asdict(self):
+        """Return folder info as a dict"""
+        return {
+            "title": self.title,
+            "uuid": self.uuid,
+            "parent": self.parent.uuid if self.parent is not None else None,
+            "subfolders": [f.uuid for f in self.subfolders],
+            "albums": [a.uuid for a in self.album_info],
+        }
+
     def __len__(self):
         """returns count of folders + albums contained in the folder"""
         return len(self.subfolders) + len(self.album_info)

osxphotos/cli/import_cli.py

@@ -25,7 +25,7 @@
 from rich.markdown import Markdown
 from strpdatetime import strpdatetime
 
-from osxphotos._constants import _OSXPHOTOS_NONE_SENTINEL
+from osxphotos._constants import _OSXPHOTOS_NONE_SENTINEL, SQLITE_CHECK_SAME_THREAD
 from osxphotos._version import __version__
 from osxphotos.cli.cli_params import TIMESTAMP_OPTION, VERBOSE_OPTION
 from osxphotos.cli.common import get_data_dir
@@ -77,7 +77,8 @@ def echo(message, emoji=True, **kwargs):
 class PhotoInfoFromFile:
     """Mock PhotoInfo class for a file to be imported
 
-    Returns None for most attributes but allows some templates like exiftool and created to work correctly"""
+    Returns None for most attributes but allows some templates like exiftool and created to work correctly
+    """
 
     def __init__(self, filepath: Union[str, Path], exiftool: Optional[str] = None):
         self._path = str(filepath)
@@ -745,7 +746,7 @@ def write_sqlite_report(
 
     file_exists = os.path.isfile(report_file)
 
-    conn = sqlite3.connect(report_file)
+    conn = sqlite3.connect(report_file, check_same_thread=SQLITE_CHECK_SAME_THREAD)
     c = conn.cursor()
 
     if not append or not file_exists:

osxphotos/cli/report_writer.py

@@ -12,6 +12,7 @@
 from contextlib import suppress
 from typing import Dict, Union
 
+from osxphotos._constants import SQLITE_CHECK_SAME_THREAD
 from osxphotos.export_db import OSXPHOTOS_ABOUT_STRING
 from osxphotos.photoexporter import ExportResults
 from osxphotos.sqlite_utils import sqlite_columns
@@ -181,7 +182,7 @@ def __init__(
             with suppress(FileNotFoundError):
                 os.unlink(self.output_file)
 
-        self._conn = sqlite3.connect(self.output_file)
+        self._conn = sqlite3.connect(self.output_file, check_same_thread=SQLITE_CHECK_SAME_THREAD)
         self._create_tables()
         self.report_id = self._generate_report_id()
 
@@ -533,7 +534,7 @@ def __init__(
             with suppress(FileNotFoundError):
                 os.unlink(self.output_file)
 
-        self._conn = sqlite3.connect(self.output_file)
+        self._conn = sqlite3.connect(self.output_file, check_same_thread=SQLITE_CHECK_SAME_THREAD)
         self._create_tables()
         self.report_id = self._generate_report_id()