Move utilities

aiida/storage/sqlite_zip/backend.py

@@ -28,10 +28,9 @@
 from aiida.storage.psql_dos.orm.querybuilder import SqlaQueryBuilder
 from aiida.storage.psql_dos.orm.utils import ModelWrapper
 from aiida.tools.archive.exceptions import ArchiveClosedError, CorruptArchive, ReadOnlyError
-from aiida.tools.archive.implementations.sqlite_zip.common import DB_FILENAME, REPO_FOLDER
 
 from . import models
-from .utils import create_sqla_engine, read_version
+from .utils import DB_FILENAME, REPO_FOLDER, create_sqla_engine, read_version
 
 
 class SqliteZipBackend(StorageBackend):  # pylint: disable=too-many-public-methods

aiida/storage/sqlite_zip/migrations/legacy_to_new.py → aiida/storage/sqlite_zip/migrations/legacy_to_main.py

@@ -7,7 +7,7 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-"""Migration from the "legacy" JSON format, to an sqlite database."""
+"""Migration from the "legacy" JSON format, to an sqlite database, and node uuid based repository to hash based."""
 from contextlib import contextmanager
 from datetime import datetime
 from hashlib import sha256
@@ -29,7 +29,6 @@
 from aiida.tools.archive.exceptions import CorruptArchive, MigrationValidationError
 
 from . import v1_db_schema as v1_schema
-from ....tools.archive.implementations.sqlite_zip.common import DB_FILENAME, META_FILENAME, REPO_FOLDER
 from .legacy.utils import update_metadata
 
 _NODE_ENTITY_NAME = 'Node'
@@ -66,6 +65,10 @@
     _LOG_ENTITY_NAME: v1_schema.DbLog,
 }
 
+_META_FILENAME = 'metadata.json'
+_DB_FILENAME = 'db.sqlite3'
+_REPO_FOLDER = 'repo'
+
 MIGRATED_TO_REVISION = 'main_0001'
 
 
@@ -103,7 +106,7 @@ def in_archive_context(_inpath):
         mode='w',
         compresslevel=compression,
         name_to_info=central_dir,
-        info_order=(META_FILENAME, DB_FILENAME)
+        info_order=(_META_FILENAME, _DB_FILENAME)
     ) as new_path:
         with in_archive_context(inpath) as path:
             length = sum(1 for _ in path.glob('**/*'))
@@ -121,18 +124,18 @@ def in_archive_context(_inpath):
                     if subpath.is_file():
                         with subpath.open('rb') as handle:
                             hashkey = chunked_file_hash(handle, sha256)
-                        if f'{REPO_FOLDER}/{hashkey}' not in central_dir:
+                        if f'{_REPO_FOLDER}/{hashkey}' not in central_dir:
                             with subpath.open('rb') as handle:
-                                with (new_path / f'{REPO_FOLDER}/{hashkey}').open(mode='wb') as handle2:
+                                with (new_path / f'{_REPO_FOLDER}/{hashkey}').open(mode='wb') as handle2:
                                     shutil.copyfileobj(handle, handle2)
                     node_repos.setdefault(uuid, []).append((posix_rel.as_posix(), hashkey))
             MIGRATE_LOGGER.report(f'Unique files written: {len(central_dir)}')
 
-        _json_to_sqlite(working / DB_FILENAME, data, node_repos)
+        _json_to_sqlite(working / _DB_FILENAME, data, node_repos)
 
         MIGRATE_LOGGER.report('Finalising archive')
-        with (working / DB_FILENAME).open('rb') as handle:
-            with (new_path / DB_FILENAME).open(mode='wb') as handle2:
+        with (working / _DB_FILENAME).open('rb') as handle:
+            with (new_path / _DB_FILENAME).open(mode='wb') as handle2:
                 shutil.copyfileobj(handle, handle2)
 
         # remove legacy keys from metadata and store
@@ -144,7 +147,7 @@ def in_archive_context(_inpath):
         metadata['key_format'] = 'sha256'
         metadata['mtime'] = datetime.now().isoformat()
         update_metadata(metadata, MIGRATED_TO_REVISION)
-        (new_path / META_FILENAME).write_text(json.dumps(metadata))
+        (new_path / _META_FILENAME).write_text(json.dumps(metadata))
 
 
 def _json_to_sqlite(

aiida/storage/sqlite_zip/migrations/main.py

@@ -22,12 +22,12 @@
 
 from aiida.common import json
 from aiida.common.progress_reporter import get_progress_reporter
+from aiida.storage.sqlite_zip.migrations.utils import copy_tar_to_zip, copy_zip_to_zip
 from aiida.tools.archive.common import MIGRATE_LOGGER
 from aiida.tools.archive.exceptions import ArchiveMigrationError, CorruptArchive
-from aiida.tools.archive.implementations.sqlite_zip.common import copy_tar_to_zip, copy_zip_to_zip
 
 from .legacy import FINAL_LEGACY_VERSION, LEGACY_MIGRATE_FUNCTIONS
-from .legacy_to_new import MIGRATED_TO_REVISION, perform_v1_migration
+from .legacy_to_main import MIGRATED_TO_REVISION, perform_v1_migration
 
 
 def _alembic_config() -> Config:

aiida/tools/archive/implementations/sqlite_zip/common.py → aiida/storage/sqlite_zip/migrations/utils.py

@@ -18,11 +18,6 @@
 
 from aiida.common.progress_reporter import create_callback, get_progress_reporter
 
-META_FILENAME = 'metadata.json'
-DB_FILENAME = 'db.sqlite3'
-# folder to store repository files in
-REPO_FOLDER = 'repo'
-
 
 def copy_zip_to_zip(
     inpath: Path,

aiida/storage/sqlite_zip/utils.py

@@ -23,6 +23,12 @@
 META_FILENAME = 'metadata.json'
 """The filename containing meta information about the storage instance."""
 
+DB_FILENAME = 'db.sqlite3'
+"""The filename of the SQLite database."""
+
+REPO_FOLDER = 'repo'
+"""The name of the folder containing the repository files."""
+
 
 def sqlite_enforce_foreign_keys(dbapi_connection, _):
     """Enforce foreign key constraints, when using sqlite backend (off by default)"""

aiida/tools/archive/implementations/sqlite_zip/writer.py

@@ -33,8 +33,6 @@
 from aiida.tools.archive.abstract import ArchiveFormatAbstract, ArchiveWriterAbstract
 from aiida.tools.archive.exceptions import CorruptArchive, IncompatibleArchiveVersionError
 
-from .common import DB_FILENAME, META_FILENAME, REPO_FOLDER
-
 
 @functools.lru_cache(maxsize=10)
 def _get_model_from_entity(entity_type: EntityTypes):
@@ -58,8 +56,8 @@ def _get_model_from_entity(entity_type: EntityTypes):
 class ArchiveWriterSqlZip(ArchiveWriterAbstract):
     """AiiDA archive writer implementation."""
 
-    meta_name = META_FILENAME
-    db_name = DB_FILENAME
+    meta_name = utils.META_FILENAME
+    db_name = utils.DB_FILENAME
 
     def __init__(
         self,
@@ -197,8 +195,8 @@ def put_object(self, stream: BinaryIO, *, buffer_size: Optional[int] = None, key
         if key is None:
             key = chunked_file_hash(stream, hashlib.sha256)
             stream.seek(0)
-        if f'{REPO_FOLDER}/{key}' not in self._central_dir:
-            self._stream_binary(f'{REPO_FOLDER}/{key}', stream, buffer_size=buffer_size)
+        if f'{utils.REPO_FOLDER}/{key}' not in self._central_dir:
+            self._stream_binary(f'{utils.REPO_FOLDER}/{key}', stream, buffer_size=buffer_size)
         return key
 
     def delete_object(self, key: str) -> None:
@@ -210,9 +208,9 @@ class ArchiveAppenderSqlZip(ArchiveWriterSqlZip):
 
     def delete_object(self, key: str) -> None:
         self._assert_in_context()
-        if f'{REPO_FOLDER}/{key}' in self._central_dir:
+        if f'{utils.REPO_FOLDER}/{key}' in self._central_dir:
             raise IOError(f'Cannot delete object {key!r} that has been added in the same append context')
-        self._deleted_paths.add(f'{REPO_FOLDER}/{key}')
+        self._deleted_paths.add(f'{utils.REPO_FOLDER}/{key}')
 
     def __enter__(self) -> 'ArchiveAppenderSqlZip':
         """Start appending to the archive"""
@@ -226,7 +224,7 @@ def __enter__(self) -> 'ArchiveAppenderSqlZip':
                 f'Archive is version {version!r} but expected {self._format.latest_version!r}'
             )
         # load the metadata
-        self._metadata = json.loads(read_file_in_zip(self._path, META_FILENAME, 'utf8', search_limit=4))
+        self._metadata = json.loads(read_file_in_zip(self._path, utils.META_FILENAME, 'utf8', search_limit=4))
         # overwrite metadata
         self._metadata['mtime'] = datetime.now().isoformat()
         self._metadata['compression'] = self._compression
@@ -247,7 +245,7 @@ def __enter__(self) -> 'ArchiveAppenderSqlZip':
         db_file = self._work_dir / self.db_name
         with db_file.open('wb') as handle:
             try:
-                extract_file_in_zip(self.path, DB_FILENAME, handle, search_limit=4)
+                extract_file_in_zip(self.path, utils.DB_FILENAME, handle, search_limit=4)
             except Exception as exc:
                 raise CorruptArchive(f'database could not be read: {exc}') from exc
         # open a connection to the database

tests/tools/archive/test_common.py → tests/tools/archive/test_utils.py

@@ -7,10 +7,10 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-"""Test common functions."""
+"""Test utility functions."""
 from archive_path import TarPath, ZipPath
 
-from aiida.tools.archive.implementations.sqlite_zip.common import copy_tar_to_zip, copy_zip_to_zip
+from aiida.storage.sqlite_zip.migrations.utils import copy_tar_to_zip, copy_zip_to_zip
 
 
 def test_copy_zip_to_zip(tmp_path):