Use __slots__ for frequently created classes

ch_backup/backup/deduplication.py

@@ -13,14 +13,24 @@
 from ch_backup.backup_context import BackupContext
 from ch_backup.clickhouse.models import Database, FrozenPart
 from ch_backup.clickhouse.schema import is_replicated
-from ch_backup.util import utcnow
+from ch_backup.util import Slotted, utcnow
 
 
-class PartDedupInfo:
+class PartDedupInfo(Slotted):
     """
     Information about data part to use for deduplication / creation incremental backups.
     """
 
+    __slots__ = (
+        "backup_path",
+        "checksum",
+        "size",
+        "files",
+        "tarball",
+        "disk_name",
+        "verified",
+    )
+
     def __init__(
         self,
         backup_path: str,
@@ -39,12 +49,6 @@ def __init__(
         self.disk_name = disk_name
         self.verified = verified
 
-    def __repr__(self):
-        return f"PartDedupInfo({self.__dict__})"
-
-    def __eq__(self, other):
-        return self.__dict__ == other.__dict__
-
 
 TableDedupInfo = Dict[str, PartDedupInfo]
 

ch_backup/backup/layout.py

@@ -593,7 +593,7 @@ def _target_part_size(self, part: PartMetadata) -> int:
         """
         Predicts tar archive size after encryption.
         """
-        tar_size = calc_tarball_size(part.raw_metadata["files"], part.size)
+        tar_size = calc_tarball_size(list(part.raw_metadata.files), part.size)
         return calc_encrypted_size(
             tar_size, self._encryption_chunk_size, self._encryption_metadata_size
         )

ch_backup/backup/metadata/part_metadata.py

@@ -2,17 +2,43 @@
 Backup metadata for ClickHouse data part.
 """
 
-from types import SimpleNamespace
 from typing import Optional, Sequence
 
 from ch_backup.clickhouse.models import FrozenPart
+from ch_backup.util import Slotted
 
 
-class PartMetadata(SimpleNamespace):
+class RawMetadata(Slotted):
+    """
+    Raw metadata for ClickHouse data part.
+    """
+
+    __slots__ = "checksum", "size", "files", "tarball", "link", "disk_name"
+
+    def __init__(
+        self,
+        checksum: str,
+        size: int,
+        files: Sequence[str],
+        tarball: bool,
+        link: str = None,
+        disk_name: str = None,
+    ) -> None:
+        self.checksum = checksum
+        self.size = size
+        self.files = files
+        self.tarball = tarball
+        self.link = link
+        self.disk_name = disk_name
+
+
+class PartMetadata(Slotted):
     """
     Backup metadata for ClickHouse data part.
     """
 
+    __slots__ = "database", "table", "name", "raw_metadata"
+
     # pylint: disable=too-many-arguments
     def __init__(
         self,
@@ -26,60 +52,54 @@ def __init__(
         link: str = None,
         disk_name: str = None,
     ) -> None:
-        super().__init__()
         self.database: str = database
         self.table: str = table
         self.name: str = name
-        self.raw_metadata: dict = {
-            "checksum": checksum,
-            "size": size,
-            "files": files,
-            "tarball": tarball,
-            "link": link,
-            "disk_name": disk_name,
-        }
+        self.raw_metadata: RawMetadata = RawMetadata(
+            checksum, size, files, tarball, link, disk_name
+        )
 
     @property
     def checksum(self) -> str:
         """
         Return data part checksum.
         """
-        return self.raw_metadata["checksum"]
+        return self.raw_metadata.checksum
 
     @property
     def size(self) -> int:
         """
         Return data part size.
         """
-        return self.raw_metadata["size"]
+        return self.raw_metadata.size
 
     @property
     def files(self) -> Sequence[str]:
         """
         Return data part files.
         """
-        return self.raw_metadata["files"]
+        return self.raw_metadata.files
 
     @property
     def link(self) -> Optional[str]:
         """
         For deduplicated data parts it returns link to the source backup (its path). Otherwise None is returned.
         """
-        return self.raw_metadata["link"]
+        return self.raw_metadata.link
 
     @property
     def disk_name(self) -> str:
         """
         Return disk name where part is stored.
         """
-        return self.raw_metadata.get("disk_name", "default")
+        return self.raw_metadata.disk_name if self.raw_metadata.disk_name else "default"
 
     @property
     def tarball(self) -> bool:
         """
         Returns true if part files stored as single tarball.
         """
-        return self.raw_metadata["tarball"]
+        return self.raw_metadata.tarball
 
     @classmethod
     def load(

ch_backup/clickhouse/models.py

@@ -7,6 +7,7 @@
 from typing import List, Optional, Tuple
 
 import ch_backup.logging
+from ch_backup.util import Slotted
 
 
 class Disk(SimpleNamespace):
@@ -163,11 +164,22 @@ def set_engine_from_sql(self, db_sql: str) -> None:
             self.engine = match.group("engine")
 
 
-class FrozenPart(SimpleNamespace):
+class FrozenPart(Slotted):
     """
     Freezed data part.
     """
 
+    __slots__ = (
+        "database",
+        "table",
+        "name",
+        "disk_name",
+        "path",
+        "checksum",
+        "size",
+        "files",
+    )
+
     def __init__(
         self,
         database: str,

ch_backup/storage/async_pipeline/stages/storage/multipart_storage_uploading_stage.py

@@ -8,16 +8,18 @@
 from ch_backup.storage.async_pipeline.base_pipeline.handler import Handler
 from ch_backup.storage.async_pipeline.stages.types import StageType
 from ch_backup.storage.engine.base import PipeLineCompatibleStorageEngine
+from ch_backup.util import Slotted
 
 
 @dataclass
-class UploadingPart:
+class UploadingPart(Slotted):
     """
     Passed between uploading stages.
     """
 
+    __slots__ = "data", "upload_id"
     data: bytes
-    upload_id: Optional[str] = None
+    upload_id: Optional[str]
 
 
 class StartMultipartUploadStage(Handler):

ch_backup/util.py

@@ -444,3 +444,23 @@ def replace_macros(string: str, macros: dict) -> str:
         pattern=r"{([^{}]+)}",
         repl=lambda m: macros.get(m.group(1), m.group(0)),
     )
+
+
+class Slotted:
+    """
+    Allow to explicitly declare data members and deny the creation of __dict__ and __weakref__.
+    The space saved over using __dict__ can be significant. Attribute lookup speed can be significantly improved as well.
+    All child classes must declare __slots__.
+    """
+
+    __slots__ = ()
+
+    def __repr__(self):
+        repr_ = [f"{attr}: {getattr(self, attr)}" for attr in self.__slots__]  # type: ignore
+        return f"{type(self).__name__}({repr_})"
+
+    def __eq__(self, other):
+        for slot in self.__slots__:  # type: ignore
+            if not getattr(self, slot) == getattr(other, slot):
+                return False
+        return True

requirements.txt

@@ -1,4 +1,4 @@
-requests>2.19
+requests<2.32.0
 boto3<1.19
 botocore<1.22
 psutil