From 5fd095afaa3d71619beac68daaefdba66442dd7d Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 12 Apr 2022 22:58:16 +0300 Subject: [PATCH 01/48] Add support for bucket replication for createBucket / updateBucket --- CHANGELOG.md | 6 + b2sdk/_v3/__init__.py | 7 ++ b2sdk/api.py | 4 + b2sdk/bucket.py | 107 +++++++++++----- b2sdk/raw_api.py | 9 ++ b2sdk/raw_simulator.py | 39 ++++-- b2sdk/replication/__init__.py | 9 ++ b2sdk/replication/setting.py | 210 ++++++++++++++++++++++++++++++++ b2sdk/replication/types.py | 19 +++ b2sdk/session.py | 5 + test/static/test_licenses.py | 3 + test/unit/bucket/test_bucket.py | 28 +++++ 12 files changed, 405 insertions(+), 41 deletions(-) create mode 100644 b2sdk/replication/__init__.py create mode 100644 b2sdk/replication/setting.py create mode 100644 b2sdk/replication/types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d96de1b1..bafc5c9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* Add basic replication support to Bucket and FileVersion + +### Fixed +* Fix license test on Windows + ## [1.15.0] - 2022-04-12 ### Changed diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index 6bbd0717a..d69fbf10d 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -202,6 +202,13 @@ from b2sdk.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER +# replication + +from b2sdk.replication.setting import ReplicationConfiguration +from b2sdk.replication.setting import ReplicationSourceConfiguration +from b2sdk.replication.setting import ReplicationRule +from b2sdk.replication.setting import ReplicationDestinationConfiguration + # other from b2sdk.b2http import B2Http diff --git a/b2sdk/api.py b/b2sdk/api.py index fe9a563ce..c6fe3a826 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -16,6 +16,7 @@ from .cache import AbstractCache from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting +from .replication.setting import ReplicationConfiguration from .exception import BucketIdNotFound, NonExistentBucket, RestrictedBucket from .file_lock import FileRetentionSetting, LegalHold from .file_version import DownloadVersionFactory, FileIdAndName, FileVersion, FileVersionFactory @@ -211,6 +212,7 @@ def create_bucket( lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): """ Create a bucket. @@ -222,6 +224,7 @@ def create_bucket( :param dict lifecycle_rules: bucket lifecycle rules to store with the bucket :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled + :param b2sdk.v2.ReplicationConfiguration replication: bucket replication rules or ``None`` :return: a Bucket object :rtype: b2sdk.v2.Bucket """ @@ -236,6 +239,7 @@ def create_bucket( lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, + replication=replication, ) bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict(self, response) assert name == bucket.name, 'API created a bucket with different name\ diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 41fe92b41..a29e1a861 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -9,27 +9,42 @@ ###################################################################### import logging + from typing import Optional, Tuple from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode -from .exception import BucketIdNotFound, CopySourceTooBig, FileNotPresent, FileOrBucketNotFound, UnexpectedCloudBehaviour, UnrecognizedBucketType +from .exception import ( + BucketIdNotFound, + CopySourceTooBig, + FileNotPresent, + FileOrBucketNotFound, + UnexpectedCloudBehaviour, + UnrecognizedBucketType, +) from .file_lock import ( + UNKNOWN_BUCKET_RETENTION, BucketRetentionSetting, FileLockConfiguration, FileRetentionSetting, - UNKNOWN_BUCKET_RETENTION, LegalHold, ) from .file_version import DownloadVersion, FileVersion from .progress import AbstractProgressListener, DoNothingProgressListener +from .replication.setting import ReplicationConfiguration from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.write_intent import WriteIntent from .transfer.inbound.downloaded_file import DownloadedFile from .transfer.outbound.copy_source import CopySource from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile -from .utils import B2TraceMeta, disable_trace, limit_trace_arguments -from .utils import b2_url_encode, validate_b2_file_name +from .utils import ( + B2TraceMeta, + b2_url_encode, + disable_trace, + limit_trace_arguments, + validate_b2_file_name, +) + logger = logging.getLogger(__name__) @@ -58,6 +73,7 @@ def __init__( ), default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): """ :param b2sdk.v2.B2Api api: an API object @@ -73,6 +89,7 @@ def __init__( :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings :param b2sdk.v2.BucketRetentionSetting default_retention: default retention setting :param bool is_file_lock_enabled: whether file locking is enabled or not + :param b2sdk.v2.ReplicationConfiguration replication: replication rules for the bucket """ self.api = api self.id_ = id_ @@ -87,6 +104,7 @@ def __init__( self.default_server_side_encryption = default_server_side_encryption self.default_retention = default_retention self.is_file_lock_enabled = is_file_lock_enabled + self.replication = replication def get_fresh_state(self) -> 'Bucket': """ @@ -132,6 +150,7 @@ def update( if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): """ Update various bucket parameters. @@ -157,6 +176,7 @@ def update( if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, + replication=replication, ) ) @@ -967,32 +987,59 @@ def from_api_bucket_dict(cls, api, bucket_dict): .. code-block:: python - { - "bucketType": "allPrivate", - "bucketId": "a4ba6a39d8b6b5fd561f0010", - "bucketName": "zsdfrtsazsdfafr", - "accountId": "4aa9865d6f00", - "bucketInfo": {}, - "options": [], - "revision": 1, - "defaultServerSideEncryption": { - "isClientAuthorizedToRead" : true, - "value": { - "algorithm" : "AES256", - "mode" : "SSE-B2" - } - }, - "fileLockConfiguration": { - "isClientAuthorizedToRead": true, - "value": { - "defaultRetention": { - "mode": null, - "period": null - }, - "isFileLockEnabled": false + { + "bucketType": "allPrivate", + "bucketId": "a4ba6a39d8b6b5fd561f0010", + "bucketName": "zsdfrtsazsdfafr", + "accountId": "4aa9865d6f00", + "bucketInfo": {}, + "options": [], + "revision": 1, + "defaultServerSideEncryption": { + "isClientAuthorizedToRead" : true, + "value": { + "algorithm" : "AES256", + "mode" : "SSE-B2" + } + }, + "fileLockConfiguration": { + "isClientAuthorizedToRead": true, + "value": { + "defaultRetention": { + "mode": null, + "period": null + }, + "isFileLockEnabled": false + } + }, + "replicationConfiguration": { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": "c5f35d53a90a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 1, + "replicationRuleName": "replication-us-west" + }, + { + "destinationBucketId": "55f34d53a96a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 2, + "replicationRuleName": "replication-us-west-2" + } + ], + "sourceApplicationKeyId": "10053d55ae26b790000000006" + }, + "asReplicationDestination": { + "sourceToDestinationKeyMapping": { + "10053d55ae26b790000000045": "10053d55ae26b790000000004", + "10053d55ae26b790000000046": "10053d55ae26b790030000004" + } } - } - } + } + } into a Bucket object. @@ -1016,6 +1063,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) + replication = ReplicationConfiguration.from_bucket_dict(bucket_dict) return cls.BUCKET_CLASS( api, bucket_id, @@ -1030,4 +1078,5 @@ def from_api_bucket_dict(cls, api, bucket_dict): default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, + replication, ) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index e597a8b9f..e5b0a3026 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -17,6 +17,7 @@ from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault, AccessDenied, SSECKeyError, RetentionWriteError from .encryption.setting import EncryptionMode, EncryptionSetting +from .replication.setting import ReplicationConfiguration from .file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from .utils import b2_url_encode from b2sdk.http_constants import FILE_INFO_HEADER_PREFIX @@ -126,6 +127,7 @@ def create_bucket( lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): pass @@ -283,6 +285,7 @@ def update_bucket( if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): pass @@ -391,6 +394,7 @@ def create_bucket( lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): kwargs = dict( accountId=account_id, @@ -410,6 +414,8 @@ def create_bucket( ] = default_server_side_encryption.serialize_to_json_for_request() if is_file_lock_enabled is not None: kwargs['fileLockEnabled'] = is_file_lock_enabled + if replication is not None: + kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() return self._post_json( api_url, 'b2_create_bucket', @@ -695,6 +701,7 @@ def update_bucket( if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): assert bucket_info is not None or bucket_type is not None @@ -716,6 +723,8 @@ def update_bucket( ] = default_server_side_encryption.serialize_to_json_for_request() if default_retention is not None: kwargs['defaultRetention'] = default_retention.serialize_to_json_for_request() + if replication is not None: + kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() return self._post_json( api_url, diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index b8ee8936b..5f708d60c 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -8,16 +8,19 @@ # ###################################################################### -from typing import Optional import collections import io import logging import random import re -import time import threading +import time + from contextlib import contextmanager +from typing import Optional +from b2sdk.http_constants import FILE_INFO_HEADER_PREFIX, HEX_DIGITS_AT_END +from b2sdk.replication.setting import ReplicationConfiguration from requests.structures import CaseInsensitiveDict from .b2http import ResponseContextManager @@ -37,21 +40,21 @@ MissingPart, NonExistentBucket, PartSha1Mismatch, + SSECKeyError, Unauthorized, UnsatisfiableRange, - SSECKeyError, ) -from .file_lock import BucketRetentionSetting, FileRetentionSetting, NO_RETENTION_BUCKET_SETTING, LegalHold -from .raw_api import AbstractRawApi, MetadataDirectiveMode, ALL_CAPABILITIES -from .utils import ( - b2_url_decode, - b2_url_encode, - ConcurrentUsedAuthTokenGuard, - hex_sha1_of_bytes, +from .file_lock import ( + NO_RETENTION_BUCKET_SETTING, + BucketRetentionSetting, + FileRetentionSetting, + LegalHold, ) -from b2sdk.http_constants import FILE_INFO_HEADER_PREFIX, HEX_DIGITS_AT_END -from .stream.hashing import StreamWithHash from .file_version import UNVERIFIED_CHECKSUM_PREFIX +from .raw_api import ALL_CAPABILITIES, AbstractRawApi, MetadataDirectiveMode +from .stream.hashing import StreamWithHash +from .utils import ConcurrentUsedAuthTokenGuard, b2_url_decode, b2_url_encode, hex_sha1_of_bytes + logger = logging.getLogger(__name__) @@ -492,6 +495,7 @@ def __init__( options_set=None, default_server_side_encryption=None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): assert bucket_type in ['allPrivate', 'allPublic'] self.api = api @@ -516,6 +520,7 @@ def __init__( self.default_server_side_encryption = default_server_side_encryption self.is_file_lock_enabled = is_file_lock_enabled self.default_retention = NO_RETENTION_BUCKET_SETTING + self.replication = replication def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketEncryption') @@ -559,6 +564,9 @@ def bucket_dict(self, account_auth_token): } # yapf: disable else: file_lock_configuration = {'isClientAuthorizedToRead': False, 'value': None} + + replication = self.replication and self.replication.as_dict() + return dict( accountId=self.account_id, bucketName=self.bucket_name, @@ -571,6 +579,7 @@ def bucket_dict(self, account_auth_token): revision=self.revision, defaultServerSideEncryption=default_sse, fileLockConfiguration=file_lock_configuration, + replicationConfiguration=replication, ) def cancel_large_file(self, file_id): @@ -925,6 +934,7 @@ def _update_bucket( if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): if if_revision_is is not None and self.revision != if_revision_is: raise Conflict() @@ -942,6 +952,7 @@ def _update_bucket( if default_retention: self.default_retention = default_retention self.revision += 1 + self.replication = replication return self.bucket_dict(self.api.current_token) def upload_file( @@ -1211,6 +1222,7 @@ def create_bucket( lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): if not re.match(r'^[-a-zA-Z0-9]*$', bucket_name): raise BadJson('illegal bucket name: ' + bucket_name) @@ -1230,6 +1242,7 @@ def create_bucket( # watch out for options! default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, + replication=replication, ) self.bucket_name_to_bucket[bucket_name] = bucket self.bucket_id_to_bucket[bucket_id] = bucket @@ -1688,6 +1701,7 @@ def update_bucket( if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): assert bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption bucket = self._get_bucket_by_id(bucket_id) @@ -1700,6 +1714,7 @@ def update_bucket( if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, + replication=replication, ) def upload_file( diff --git a/b2sdk/replication/__init__.py b/b2sdk/replication/__init__.py new file mode 100644 index 000000000..9f62b3c8e --- /dev/null +++ b/b2sdk/replication/__init__.py @@ -0,0 +1,9 @@ +###################################################################### +# +# File: b2sdk/replication/__init__.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py new file mode 100644 index 000000000..1d6475420 --- /dev/null +++ b/b2sdk/replication/setting.py @@ -0,0 +1,210 @@ +###################################################################### +# +# File: b2sdk/replication/setting.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import re + +from builtins import classmethod +from dataclasses import dataclass +from typing import ClassVar, Dict, List, Optional + + +@dataclass +class ReplicationRule: + """ + Hold information about replication rule: destination bucket, priority, + prefix and rule name. + """ + + destination_bucket_id: str + replication_rule_name: str + file_name_prefix: str = '' + is_enabled: bool = True + priority: int = 1 + + REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') + + def __post_init__(self): + if not self.destination_bucket_id: + raise ValueError('destination_bucket_id is required') + + # TODO + # if not (1 < self.priority < 255): + # raise ValueError() + + # TODO: file_name_prefix validation + + if not self.REPLICATION_RULE_REGEX.match(self.replication_rule_name): + raise ValueError('replication_rule_name is invalid') + + def as_dict(self) -> dict: + return { + 'destinationBucketId': self.destination_bucket_id, + 'fileNamePrefix': self.file_name_prefix, + 'isEnabled': self.is_enabled, + 'priority': self.priority, + 'replicationRuleName': self.replication_rule_name, + } + + @classmethod + def from_dict(cls, value_dict: dict) -> 'ReplicationRule': + return cls( + destination_bucket_id=value_dict['destinationBucketId'], + file_name_prefix=value_dict['fileNamePrefix'], + is_enabled=value_dict['isEnabled'], + priority=value_dict['priority'], + replication_rule_name=value_dict['replicationRuleName'], + ) + + +@dataclass +class ReplicationSourceConfiguration: + """ + Hold information about bucket as replication source + """ + + replication_rules: List[ReplicationRule] + source_application_key_id: str + + def __post_init__(self): + if not self.replication_rules: + raise ValueError("replication_rules must not be empty") + + if not self.source_application_key_id: + raise ValueError("source_application_key_id must not be empty") + + def as_dict(self) -> dict: + return { + "replicationRules": [rule.as_dict() for rule in self.replication_rules], + "sourceApplicationKeyId": self.source_application_key_id, + } + + @classmethod + def from_dict(cls, value_dict: dict) -> 'ReplicationSourceConfiguration': + return cls( + replication_rules=[ + ReplicationRule.from_dict(rule_dict) + for rule_dict in value_dict['replicationRules'] + ], + source_application_key_id=value_dict['sourceApplicationKeyId'], + ) + + +@dataclass +class ReplicationDestinationConfiguration: + source_to_destination_key_mapping: Dict[str, str] + + def __post_init__(self): + if not self.source_to_destination_key_mapping: + raise ValueError("source_to_destination_key_mapping must not be empty") + + for source, destination in self.source_to_destination_key_mapping.items(): + if not source or not destination: + raise ValueError("source_to_destination_key_mapping must not contain \ + empty keys or values: ({}, {})".format(source, destination)) + + def as_dict(self) -> dict: + return { + 'sourceToDestinationKeyMapping': self.source_to_destination_key_mapping, + } + + @classmethod + def from_dict(cls, value_dict: dict) -> 'ReplicationDestinationConfiguration': + return cls( + source_to_destination_key_mapping=value_dict['sourceToDestinationKeyMapping'], + ) + + +@dataclass +class ReplicationConfiguration: + """ + Hold information about bucket replication + """ + + as_replication_source: Optional[ReplicationSourceConfiguration] = None + as_replication_destination: Optional[ReplicationDestinationConfiguration] = None + + def __post_init__(self): + if not self.as_replication_source and not self.as_replication_destination: + raise ValueError( + "Must provide either as_replication_source or as_replication_destination" + ) + + def serialize_to_json_for_request(self) -> dict: + return self.as_dict() + + def as_dict(self) -> dict: + """ + Represent the setting as a dict, for example: + + .. code-block:: python + + { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": "c5f35d53a90a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 1, + "replicationRuleName": "replication-us-west" + }, + { + "destinationBucketId": "55f34d53a96a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 2, + "replicationRuleName": "replication-us-west-2" + } + ], + "sourceApplicationKeyId": "10053d55ae26b790000000006" + }, + "asReplicationDestination": { + "sourceToDestinationKeyMapping": { + "10053d55ae26b790000000045": "10053d55ae26b790000000004", + "10053d55ae26b790000000046": "10053d55ae26b790030000004" + } + } + } + + """ + + result = {} + + if self.as_replication_source: + result['asReplicationSource'] = self.as_replication_source.as_dict() + + if self.as_replication_destination: + result['asReplicationDestination'] = self.as_replication_destination.as_dict() + + return result + + @classmethod + def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfiguration']: + """ + Returns ReplicationConfiguration for the given bucket dict retrieved from the api, or None if no replication configured. + """ + replication_data = bucket_dict.get('replicationConfiguration') + if replication_data is None: + return + + return cls.from_dict(bucket_dict['replicationConfiguration']) + + @classmethod + def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': + replication_source_dict = value_dict.get('asReplicationSource') + as_replication_source = replication_source_dict and ReplicationSourceConfiguration.from_dict(replication_source_dict) + + replication_destination_dict = value_dict.get('asReplicationDestination') + as_replication_destination = replication_destination_dict and ReplicationDestinationConfiguration.from_dict(replication_destination_dict) + + return cls( + as_replication_source=as_replication_source, + as_replication_destination=as_replication_destination, + ) diff --git a/b2sdk/replication/types.py b/b2sdk/replication/types.py new file mode 100644 index 000000000..687f88f54 --- /dev/null +++ b/b2sdk/replication/types.py @@ -0,0 +1,19 @@ +###################################################################### +# +# File: b2sdk/replication/types.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from enum import Enum, unique + + +@unique +class ReplicationStatus(Enum): + PENDING = 'PENDING' + COMPLETED = 'COMPLETED' + FAILED = 'FAILED' + REPLICA = 'REPLICA' diff --git a/b2sdk/session.py b/b2sdk/session.py index 01ea58e96..7cb5e0645 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -19,6 +19,7 @@ from b2sdk.b2http import B2Http from b2sdk.cache import AbstractCache, AuthInfoCache, DummyCache from b2sdk.encryption.setting import EncryptionSetting +from b2sdk.replication.setting import ReplicationConfiguration from b2sdk.exception import (InvalidAuthToken, Unauthorized) from b2sdk.file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from b2sdk.raw_api import ALL_CAPABILITIES, REALM_URLS @@ -146,6 +147,7 @@ def create_bucket( lifecycle_rules=None, default_server_side_encryption=None, is_file_lock_enabled: Optional[bool] = None, + replication: Optional[ReplicationConfiguration] = None, ): return self._wrap_default_token( self.raw_api.create_bucket, @@ -157,6 +159,7 @@ def create_bucket( lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, + replication=replication, ) def create_key( @@ -316,6 +319,7 @@ def update_bucket( if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, + replication: Optional[ReplicationConfiguration] = None, ): return self._wrap_default_token( self.raw_api.update_bucket, @@ -328,6 +332,7 @@ def update_bucket( if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, + replication=replication, ) def upload_file( diff --git a/test/static/test_licenses.py b/test/static/test_licenses.py index babda3306..f7b6fc186 100644 --- a/test/static/test_licenses.py +++ b/test/static/test_licenses.py @@ -17,6 +17,9 @@ def test_files_headers(): for file in glob('**/*.py', recursive=True): with open(file) as fd: + file = file.replace( + '\\', '/' + ) # glob('**/*.py') on Windows returns "b2\bucket.py" (wrong slash) head = ''.join(islice(fd, 9)) if 'All Rights Reserved' not in head: pytest.fail('Missing "All Rights Reserved" in the header in: {}'.format(file)) diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 29d981294..c37824b71 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -58,6 +58,8 @@ from apiver_deps import CopySource, UploadSourceLocalFile, WriteIntent from apiver_deps import BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, RetentionPeriod, \ NO_RETENTION_FILE_SETTING +from apiver_deps import ReplicationConfiguration, ReplicationSourceConfiguration, \ + ReplicationRule, ReplicationDestinationConfiguration pytestmark = [pytest.mark.apiver(from_ver=1)] @@ -86,6 +88,30 @@ algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(key_id=None, secret=None), ) +REPLICATION = ReplicationConfiguration( + as_replication_source=ReplicationSourceConfiguration( + replication_rules=[ + ReplicationRule( + destination_bucket_id='c5f35d53a90a7ea284fb0719', + replication_rule_name='replication-us-west', + ), + ReplicationRule( + destination_bucket_id='55f34d53a96a7ea284fb0719', + replication_rule_name='replication-us-west-2', + file_name_prefix='replica/', + is_enabled=False, + priority=255, + ), + ], + source_application_key_id='10053d55ae26b790000000006', + ), + as_replication_destination=ReplicationDestinationConfiguration( + source_to_destination_key_mapping={ + "10053d55ae26b790000000045": "10053d55ae26b790000000004", + "10053d55ae26b790000000046": "10053d55ae26b790030000004" + }, + ), +) def write_file(path, data): @@ -935,6 +961,7 @@ def test_update(self): default_retention=BucketRetentionSetting( RetentionMode.COMPLIANCE, RetentionPeriod(years=7) ), + replication=REPLICATION, ) if apiver_deps.V <= 1: self.assertEqual( @@ -992,6 +1019,7 @@ def test_update(self): 'options_set': set(), 'default_server_side_encryption': SSE_B2_AES, 'default_retention': BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=7)), + 'replication': REPLICATION, } for attr_name, attr_value in assertions_mapping.items(): self.assertEqual(attr_value, getattr(result, attr_name), attr_name) From ceff24cf756187ae70d215723ad1481ba638e671 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 14 Apr 2022 01:15:40 +0300 Subject: [PATCH 02/48] Add replication_status field to files --- b2sdk/api.py | 2 +- b2sdk/bucket.py | 1 - b2sdk/file_version.py | 17 +++++++++++++++-- b2sdk/raw_simulator.py | 7 ++++++- b2sdk/replication/setting.py | 24 ++++++++++++++---------- b2sdk/replication/types.py | 1 + 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index c6fe3a826..72f30fcef 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -535,7 +535,7 @@ def get_file_info(self, file_id: str) -> FileVersion: """ Gets info about file version. - :param str file_id: the id of the file who's info will be retrieved. + :param str file_id: the id of the file whose info will be retrieved. """ return self.file_version_factory.from_api_response( self.session.get_file_info_by_id(file_id) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index a29e1a861..a0447323a 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -45,7 +45,6 @@ validate_b2_file_name, ) - logger = logging.getLogger(__name__) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 4a04db340..d4f8a9f66 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -11,6 +11,7 @@ from typing import Dict, Optional, Union, Tuple, TYPE_CHECKING from .encryption.setting import EncryptionSetting, EncryptionSettingFactory +from .replication.types import ReplicationStatus from .http_constants import FILE_INFO_HEADER_PREFIX_LOWER, SRC_LAST_MODIFIED_MILLIS from .file_lock import FileRetentionSetting, LegalHold, NO_RETENTION_FILE_SETTING from .progress import AbstractProgressListener @@ -191,6 +192,7 @@ class FileVersion(BaseFileVersion): 'bucket_id', 'content_md5', 'action', + 'replication_status', ] def __init__( @@ -210,11 +212,13 @@ def __init__( server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, + replication_status: ReplicationStatus = None, ): self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.action = action + self.replication_status = replication_status super().__init__( api=api, @@ -238,6 +242,7 @@ def _get_args_for_clone(self): 'bucket_id': self.bucket_id, 'action': self.action, 'content_md5': self.content_md5, + 'replication_status': self.replication_status, } ) return args @@ -246,6 +251,7 @@ def as_dict(self): result = super().as_dict() result['accountId'] = self.account_id result['bucketId'] = self.bucket_id + result['replicationStatus'] = self.replication_status and self.replication_status.value if self.action is not None: result['action'] = self.action @@ -369,7 +375,8 @@ def from_api_response(self, file_version_dict, force_action=None): "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", "fileName": "randomdata", "size": 0, - "uploadTimestamp": 1451444477000 + "uploadTimestamp": 1451444477000, + "replicationStatus": "PENDING" } or this: @@ -385,7 +392,8 @@ def from_api_response(self, file_version_dict, force_action=None): "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", "fileInfo": {}, "fileName": "randomdata", - "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"} + "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, + "replicationStatus": "COMPLETED" } into a :py:class:`b2sdk.v2.FileVersion` object. @@ -411,6 +419,10 @@ def from_api_response(self, file_version_dict, force_action=None): file_retention = FileRetentionSetting.from_file_version_dict(file_version_dict) legal_hold = LegalHold.from_file_version_dict(file_version_dict) + + replication_status_value = file_version_dict['replicationStatus'] + replication_status = replication_status_value and ReplicationStatus[replication_status_value] + return self.FILE_VERSION_CLASS( self.api, id_, @@ -427,6 +439,7 @@ def from_api_response(self, file_version_dict, force_action=None): server_side_encryption, file_retention, legal_hold, + replication_status, ) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 5f708d60c..41a473699 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -25,6 +25,7 @@ from .b2http import ResponseContextManager from .encryption.setting import EncryptionMode, EncryptionSetting +from .replication.types import ReplicationStatus from .exception import ( BadJson, BadRequest, @@ -55,7 +56,6 @@ from .stream.hashing import StreamWithHash from .utils import ConcurrentUsedAuthTokenGuard, b2_url_decode, b2_url_encode, hex_sha1_of_bytes - logger = logging.getLogger(__name__) @@ -171,6 +171,7 @@ def __init__( server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: LegalHold = LegalHold.UNSET, + replication_status: Optional[ReplicationStatus] = None, ): if action == 'hide': assert server_side_encryption is None @@ -194,6 +195,7 @@ def __init__( self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold if legal_hold is not None else LegalHold.UNSET + self.replication_status = replication_status if action == 'start': self.parts = [] @@ -290,6 +292,7 @@ def as_upload_result(self, account_auth_token): fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, + replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' @@ -310,6 +313,7 @@ def as_list_files_dict(self, account_auth_token): fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, + replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' @@ -333,6 +337,7 @@ def as_start_large_file_result(self, account_auth_token): contentType=self.content_type, fileInfo=self.file_info, uploadTimestamp=self.upload_timestamp, + replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 1d6475420..b4cb3ddd0 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -89,8 +89,7 @@ def as_dict(self) -> dict: def from_dict(cls, value_dict: dict) -> 'ReplicationSourceConfiguration': return cls( replication_rules=[ - ReplicationRule.from_dict(rule_dict) - for rule_dict in value_dict['replicationRules'] + ReplicationRule.from_dict(rule_dict) for rule_dict in value_dict['replicationRules'] ], source_application_key_id=value_dict['sourceApplicationKeyId'], ) @@ -106,8 +105,10 @@ def __post_init__(self): for source, destination in self.source_to_destination_key_mapping.items(): if not source or not destination: - raise ValueError("source_to_destination_key_mapping must not contain \ - empty keys or values: ({}, {})".format(source, destination)) + raise ValueError( + "source_to_destination_key_mapping must not contain \ + empty keys or values: ({}, {})".format(source, destination) + ) def as_dict(self) -> dict: return { @@ -116,9 +117,7 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationDestinationConfiguration': - return cls( - source_to_destination_key_mapping=value_dict['sourceToDestinationKeyMapping'], - ) + return cls(source_to_destination_key_mapping=value_dict['sourceToDestinationKeyMapping']) @dataclass @@ -188,7 +187,8 @@ def as_dict(self) -> dict: @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfiguration']: """ - Returns ReplicationConfiguration for the given bucket dict retrieved from the api, or None if no replication configured. + Returns ReplicationConfiguration for the given bucket dict retrieved from the api, + or None if no replication configured. """ replication_data = bucket_dict.get('replicationConfiguration') if replication_data is None: @@ -199,10 +199,14 @@ def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurati @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': replication_source_dict = value_dict.get('asReplicationSource') - as_replication_source = replication_source_dict and ReplicationSourceConfiguration.from_dict(replication_source_dict) + as_replication_source = replication_source_dict and ReplicationSourceConfiguration.from_dict( + replication_source_dict + ) replication_destination_dict = value_dict.get('asReplicationDestination') - as_replication_destination = replication_destination_dict and ReplicationDestinationConfiguration.from_dict(replication_destination_dict) + as_replication_destination = replication_destination_dict and ReplicationDestinationConfiguration.from_dict( + replication_destination_dict + ) return cls( as_replication_source=as_replication_source, diff --git a/b2sdk/replication/types.py b/b2sdk/replication/types.py index 687f88f54..ef4ecd4a8 100644 --- a/b2sdk/replication/types.py +++ b/b2sdk/replication/types.py @@ -17,3 +17,4 @@ class ReplicationStatus(Enum): COMPLETED = 'COMPLETED' FAILED = 'FAILED' REPLICA = 'REPLICA' + From 7e4f815ab98dd6e9c8ddfd0416294eb37cb4c460 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 16 Apr 2022 18:15:42 +0700 Subject: [PATCH 03/48] Fix apiver v1 tests --- b2sdk/v1/bucket.py | 12 +++++++++++- b2sdk/v1/file_version.py | 9 +++++++++ test/unit/api/test_api.py | 5 ++++- test/unit/bucket/test_bucket.py | 4 ++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/b2sdk/v1/bucket.py b/b2sdk/v1/bucket.py index f7d8b9ce0..b5c3845aa 100644 --- a/b2sdk/v1/bucket.py +++ b/b2sdk/v1/bucket.py @@ -8,10 +8,12 @@ # ###################################################################### +from contextlib import suppress +from typing import Optional, overload, Tuple + from .download_dest import AbstractDownloadDestination from .file_metadata import FileMetadata from .file_version import FileVersionInfo, FileVersionInfoFactory, file_version_info_from_download_version -from typing import Optional, overload, Tuple from b2sdk import v2 from b2sdk.utils import validate_b2_file_name @@ -207,6 +209,7 @@ def update( if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[v2.EncryptionSetting] = None, default_retention: Optional[v2.BucketRetentionSetting] = None, + **kwargs ): """ Update various bucket parameters. @@ -219,6 +222,13 @@ def update( :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting """ + # allow common tests to execute without hitting attributeerror + + with suppress(KeyError): + del kwargs['replication'] + self.replication = None + assert not kwargs # after we get rid of everything we don't support in this apiver, this should be empty + account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( account_id, diff --git a/b2sdk/v1/file_version.py b/b2sdk/v1/file_version.py index 83e682c75..3bc17b39a 100644 --- a/b2sdk/v1/file_version.py +++ b/b2sdk/v1/file_version.py @@ -8,6 +8,7 @@ # ###################################################################### +from contextlib import suppress from typing import Optional import datetime import functools @@ -42,6 +43,7 @@ def __init__( file_retention: Optional[v2.FileRetentionSetting] = None, legal_hold: Optional[v2.LegalHold] = None, api: Optional['v1api.B2Api'] = None, + **kwargs, ): self.id_ = id_ self.file_name = file_name @@ -59,6 +61,13 @@ def __init__( self.file_retention = file_retention self._api = api + # allow common tests to execute without hitting attributeerror + + with suppress(KeyError): + del kwargs['replication_status'] + self.replication_status = None + assert not kwargs # after we get rid of everything we don't support in this apiver, this should be empty + if v2.SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[v2.SRC_LAST_MODIFIED_MILLIS]) else: diff --git a/test/unit/api/test_api.py b/test/unit/api/test_api.py index f87ad41a2..c2f72a255 100644 --- a/test/unit/api/test_api.py +++ b/test/unit/api/test_api.py @@ -11,8 +11,8 @@ import time import pytest +from contextlib import suppress from unittest import mock - from ..test_base import create_key import apiver_deps @@ -58,6 +58,9 @@ def test_get_file_info(self): result = self.api.get_file_info(created_file.id_) if apiver_deps.V <= 1: + self.maxDiff = None + with suppress(KeyError): + del result['replicationStatus'] assert result == { 'accountId': 'account-0', 'action': 'upload', diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index c37824b71..3f1b4e3f5 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -8,6 +8,7 @@ # ###################################################################### import io +from contextlib import suppress from io import BytesIO import os import platform @@ -964,6 +965,9 @@ def test_update(self): replication=REPLICATION, ) if apiver_deps.V <= 1: + self.maxDiff = None + with suppress(KeyError): + del result['replicationConfiguration'] self.assertEqual( { 'accountId': 'account-0', From c9c69bd077fa2ccc66dcdb02d421fc9799c91f45 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 16 Apr 2022 18:39:39 +0700 Subject: [PATCH 04/48] Fix apiver v0 tests --- test/unit/api/test_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/api/test_api.py b/test/unit/api/test_api.py index c2f72a255..56a6e632a 100644 --- a/test/unit/api/test_api.py +++ b/test/unit/api/test_api.py @@ -134,6 +134,9 @@ def test_list_buckets(self, expected_delete_bucket_output): self.api.create_bucket('bucket1', 'allPrivate') bucket2 = self.api.create_bucket('bucket2', 'allPrivate') delete_output = self.api.delete_bucket(bucket2) + if expected_delete_bucket_output is not None: + with suppress(KeyError): + del delete_output['replicationConfiguration'] assert delete_output == expected_delete_bucket_output self.api.create_bucket('bucket3', 'allPrivate') assert [b.name for b in self.api.list_buckets()] == ['bucket1', 'bucket3'] From 68a5471a13ddf62244ee39effe972636853d3c13 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 16 Apr 2022 22:28:55 +0700 Subject: [PATCH 05/48] Fix FileVersionFactory for pre-replication objects --- b2sdk/file_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index d4f8a9f66..ab174c08b 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -420,7 +420,7 @@ def from_api_response(self, file_version_dict, force_action=None): legal_hold = LegalHold.from_file_version_dict(file_version_dict) - replication_status_value = file_version_dict['replicationStatus'] + replication_status_value = file_version_dict.get('replicationStatus') replication_status = replication_status_value and ReplicationStatus[replication_status_value] return self.FILE_VERSION_CLASS( From ad50ce47dd06e47aee5cf2eebd62aa86ee063692 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 17 Apr 2022 00:52:03 +0700 Subject: [PATCH 06/48] Allow for empty ReplicationConfiguration objects --- b2sdk/replication/setting.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index b4cb3ddd0..5b0b25d0d 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -129,12 +129,6 @@ class ReplicationConfiguration: as_replication_source: Optional[ReplicationSourceConfiguration] = None as_replication_destination: Optional[ReplicationDestinationConfiguration] = None - def __post_init__(self): - if not self.as_replication_source and not self.as_replication_destination: - raise ValueError( - "Must provide either as_replication_source or as_replication_destination" - ) - def serialize_to_json_for_request(self) -> dict: return self.as_dict() From bdfba93f0d85117ac0e597a9f334ca7a74374995 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sun, 17 Apr 2022 03:45:50 +0700 Subject: [PATCH 07/48] Documentation updates for replication --- b2sdk/replication/setting.py | 4 ++-- b2sdk/v1/file_version.py | 2 +- doc/source/api_types.rst | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 5b0b25d0d..f46f10b52 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -66,7 +66,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationRule': @dataclass class ReplicationSourceConfiguration: """ - Hold information about bucket as replication source + Hold information about bucket being a replication source """ replication_rules: List[ReplicationRule] @@ -123,7 +123,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationDestinationConfiguration': @dataclass class ReplicationConfiguration: """ - Hold information about bucket replication + Hold information about bucket replication configuration """ as_replication_source: Optional[ReplicationSourceConfiguration] = None diff --git a/b2sdk/v1/file_version.py b/b2sdk/v1/file_version.py index 3bc17b39a..39c2c82e2 100644 --- a/b2sdk/v1/file_version.py +++ b/b2sdk/v1/file_version.py @@ -43,7 +43,7 @@ def __init__( file_retention: Optional[v2.FileRetentionSetting] = None, legal_hold: Optional[v2.LegalHold] = None, api: Optional['v1api.B2Api'] = None, - **kwargs, + **kwargs ): self.id_ = id_ self.file_name = file_name diff --git a/doc/source/api_types.rst b/doc/source/api_types.rst index 3fbb21adf..bc37fe4b8 100644 --- a/doc/source/api_types.rst +++ b/doc/source/api_types.rst @@ -16,7 +16,7 @@ Semantic versioning Therefore when setting up **b2sdk** as a dependency, please make sure to match the version appropriately, for example you could put this in your ``requirements.txt`` to make sure your code is compatible with the ``b2sdk`` version your user will get from pypi:: - b2sdk>=1.0.0,<2.0.0 + b2sdk>=1.15.0,<2.0.0 .. _interface_versions: @@ -26,7 +26,7 @@ Interface versions You might notice that the import structure provided in the documentation looks a little odd: ``from b2sdk.v2 import ...``. The ``.v2`` part is used to keep the interface fluid without risk of breaking applications that use the old signatures. -With new versions, **b2sdk** will provide functions with signatures matching the old ones, wrapping the new interface in place of the old one. What this means for a developer using **b2sdk**, is that it will just keep working. We havealready deleted some legacy functions when moving from ``.v0`` to ``.v1``, providing equivalent wrappers to reduce the migration effort for applications using pre-1.0 versions of **b2sdk** to fixing imports. +With new versions, **b2sdk** will provide functions with signatures matching the old ones, wrapping the new interface in place of the old one. What this means for a developer using **b2sdk**, is that it will just keep working. We have already deleted some legacy functions when moving from ``.v0`` to ``.v1``, providing equivalent wrappers to reduce the migration effort for applications using pre-1.0 versions of **b2sdk** to fixing imports. It also means that **b2sdk** developers may change the interface in the future and will not need to maintain many branches and backport fixes to keep compatibility of for users of those old branches. @@ -46,7 +46,7 @@ The exception hierarchy may change in a backwards compatible manner and the deve Extensions ========== -Even in the same interface version, objects/classes/enums can get additional fields and their representations such as ``to_dict()`` or ``__repr__`` (but not ``__str__``) may start to contain those fields. +Even in the same interface version, objects/classes/enums can get additional fields and their representations such as ``as_dict()`` or ``__repr__`` (but not ``__str__``) may start to contain those fields. Methods and functions can start accepting new **optional** arguments. New methods can be added to existing classes. @@ -65,6 +65,9 @@ This should be used in 99% of use cases, it's enough to implement anything from Those modules will generally not change in a backwards-incompatible way between non-major versions. Please see :ref:`interface version compatibility ` chapter for notes on what changes must be expected. +.. note:: + Replication is currently in a Closed Beta state, where not all B2 accounts have access to the feature. The interface of the beta server API might change and the interface of **b2sdk** around replication may change as well. For the avoidance of doubt, until this message is removed, replication-related functionality of **b2sdk** should be considered as internal interface. + .. hint:: If the current version of **b2sdk** is ``4.5.6`` and you only use the *public* interface, put this in your ``requirements.txt`` to be safe:: From 3ecd1d0d840c073121c2b0fc49911ad41a3c3bf3 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Sun, 17 Apr 2022 14:27:57 +0300 Subject: [PATCH 08/48] Add test for replication setup --- b2sdk/replication/types.py | 1 - test/integration/test_raw_api.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/b2sdk/replication/types.py b/b2sdk/replication/types.py index ef4ecd4a8..687f88f54 100644 --- a/b2sdk/replication/types.py +++ b/b2sdk/replication/types.py @@ -17,4 +17,3 @@ class ReplicationStatus(Enum): COMPLETED = 'COMPLETED' FAILED = 'FAILED' REPLICA = 'REPLICA' - diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index c021d9881..fdaffa2d8 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -19,6 +19,7 @@ from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting +from b2sdk.replication.setting import ReplicationConfiguration, ReplicationSourceConfiguration, ReplicationRule from b2sdk.file_lock import BucketRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod from b2sdk.raw_api import B2RawHTTPApi, REALM_URLS from b2sdk.utils import hex_sha1_of_stream @@ -137,6 +138,28 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] + print('b2_create_bucket/replication') + # in order to test replication, we need to create a second bucket + _ = raw_api.create_bucket( + api_url, + account_auth_token, + account_id, + bucket_name + '-rep', + 'allPublic', + is_file_lock_enabled=True, + replication=ReplicationConfiguration( + as_replication_source=ReplicationSourceConfiguration( + replication_rules=[ + ReplicationRule( + destination_bucket_id=bucket_id, + replication_rule_name='test-rule', + ), + ], + source_application_key_id=key_dict['applicationKeyId'], + ), + ), + ) + ################## print('b2_update_bucket') sse_b2_aes = EncryptionSetting( From 8b3328eca28c8bc6eedde824c8c0a72ada5d7561 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 18 Apr 2022 00:16:44 +0300 Subject: [PATCH 09/48] Add check that replicationConfiguration is saved by the API --- test/integration/test_raw_api.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index fdaffa2d8..0b2693d3d 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -140,7 +140,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): print('b2_create_bucket/replication') # in order to test replication, we need to create a second bucket - _ = raw_api.create_bucket( + bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, @@ -159,6 +159,22 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): ), ), ) + assert 'replicationConfiguration' in bucket_dict + assert bucket_dict['replicationConfiguration'] == { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": bucket_id, + "fileNamePrefix": "", + "isEnabled": True, + "priority": 1, + "replicationRuleName": "test-rule" + }, + ], + "sourceApplicationKeyId": key_dict['applicationKeyId'] + }, + "asReplicationDestination": None, + } ################## print('b2_update_bucket') From b90678232c4306bafb9ac6668f95dc5bbff15425 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 18 Apr 2022 01:19:04 +0300 Subject: [PATCH 10/48] Add ReplicationConfigurationResponse class support --- b2sdk/_v3/__init__.py | 1 + b2sdk/bucket.py | 4 ++-- b2sdk/raw_simulator.py | 5 ++++- b2sdk/replication/setting.py | 37 +++++++++++++++++++++----------- test/integration/test_raw_api.py | 36 +++++++++++++++++-------------- test/unit/bucket/test_bucket.py | 7 ++++-- 6 files changed, 57 insertions(+), 33 deletions(-) diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index d69fbf10d..b02871411 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -204,6 +204,7 @@ # replication +from b2sdk.replication.setting import ReplicationConfigurationResponse from b2sdk.replication.setting import ReplicationConfiguration from b2sdk.replication.setting import ReplicationSourceConfiguration from b2sdk.replication.setting import ReplicationRule diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index a0447323a..b51058fc9 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -31,7 +31,7 @@ ) from .file_version import DownloadVersion, FileVersion from .progress import AbstractProgressListener, DoNothingProgressListener -from .replication.setting import ReplicationConfiguration +from .replication.setting import ReplicationConfiguration, ReplicationConfigurationResponse from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.write_intent import WriteIntent from .transfer.inbound.downloaded_file import DownloadedFile @@ -1062,7 +1062,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) - replication = ReplicationConfiguration.from_bucket_dict(bucket_dict) + replication = ReplicationConfigurationResponse.from_bucket_dict(bucket_dict) return cls.BUCKET_CLASS( api, bucket_id, diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 41a473699..7e2521c85 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -570,7 +570,10 @@ def bucket_dict(self, account_auth_token): else: file_lock_configuration = {'isClientAuthorizedToRead': False, 'value': None} - replication = self.replication and self.replication.as_dict() + replication = self.replication and { + 'isClientAuthorizedToRead': True, + 'value': self.replication.as_dict(), + } return dict( accountId=self.account_id, diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index f46f10b52..a710f2046 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -178,18 +178,6 @@ def as_dict(self) -> dict: return result - @classmethod - def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfiguration']: - """ - Returns ReplicationConfiguration for the given bucket dict retrieved from the api, - or None if no replication configured. - """ - replication_data = bucket_dict.get('replicationConfiguration') - if replication_data is None: - return - - return cls.from_dict(bucket_dict['replicationConfiguration']) - @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': replication_source_dict = value_dict.get('asReplicationSource') @@ -206,3 +194,28 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': as_replication_source=as_replication_source, as_replication_destination=as_replication_destination, ) + + +@dataclass +class ReplicationConfigurationResponse: + is_client_authorized_to_read: bool + value: ReplicationConfiguration + + @classmethod + def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurationResponse']: + """ + Returns ReplicationConfigurationResponse for the given bucket dict + retrieved from the api, or None if no replication configured. + """ + replication_data = bucket_dict.get('replicationConfiguration') + if replication_data is None: + return + + return cls.from_dict(bucket_dict['replicationConfiguration']) + + @classmethod + def from_dict(cls, value_dict: dict) -> 'ReplicationConfigurationResponse': + return cls( + is_client_authorized_to_read=value_dict['isClientAuthorizedToRead'], + value=ReplicationConfiguration.from_dict(value_dict['value']), + ) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 0b2693d3d..5699df21a 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -138,9 +138,10 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] - print('b2_create_bucket/replication') + # b2_create_bucket/replication-source + print('b2_create_bucket/replication-source') # in order to test replication, we need to create a second bucket - bucket_dict = raw_api.create_bucket( + replication_source_bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, @@ -159,21 +160,24 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): ), ), ) - assert 'replicationConfiguration' in bucket_dict - assert bucket_dict['replicationConfiguration'] == { - "asReplicationSource": { - "replicationRules": [ - { - "destinationBucketId": bucket_id, - "fileNamePrefix": "", - "isEnabled": True, - "priority": 1, - "replicationRuleName": "test-rule" - }, - ], - "sourceApplicationKeyId": key_dict['applicationKeyId'] + assert 'replicationConfiguration' in replication_source_bucket_dict + assert replication_source_bucket_dict['replicationConfiguration'] == { + 'isClientAuthorizedToRead': True, + 'value': { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": bucket_id, + "fileNamePrefix": "", + "isEnabled": True, + "priority": 1, + "replicationRuleName": "test-rule" + }, + ], + "sourceApplicationKeyId": key_dict['applicationKeyId'] + }, + "asReplicationDestination": None, }, - "asReplicationDestination": None, } ################## diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 3f1b4e3f5..fa695495d 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -60,7 +60,7 @@ from apiver_deps import BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, RetentionPeriod, \ NO_RETENTION_FILE_SETTING from apiver_deps import ReplicationConfiguration, ReplicationSourceConfiguration, \ - ReplicationRule, ReplicationDestinationConfiguration + ReplicationRule, ReplicationDestinationConfiguration, ReplicationConfigurationResponse pytestmark = [pytest.mark.apiver(from_ver=1)] @@ -1023,7 +1023,10 @@ def test_update(self): 'options_set': set(), 'default_server_side_encryption': SSE_B2_AES, 'default_retention': BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=7)), - 'replication': REPLICATION, + 'replication': ReplicationConfigurationResponse( + is_client_authorized_to_read=True, + value=REPLICATION, + ), } for attr_name, attr_value in assertions_mapping.items(): self.assertEqual(attr_value, getattr(result, attr_name), attr_name) From 743f558eda5ee994ec6d07c75d8bc2a44f7182da Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 18 Apr 2022 14:25:21 +0200 Subject: [PATCH 11/48] Add `is_master_key()` method to `AbstractAccountInfo` --- CHANGELOG.md | 7 +- b2sdk/account_info/abstract.py | 3 + b2sdk/replication/setting.py | 4 ++ test/integration/test_download.py | 114 +++++++++++++++--------------- 4 files changed, 68 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bafc5c9fb..bcdaba21d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -* Add basic replication support to Bucket and FileVersion +* Add basic replication support to `Bucket` and `FileVersion` +* Add `is_master_key()` method to `AbstractAccountInfo` ### Fixed * Fix license test on Windows @@ -28,9 +29,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add a parameter to customize `sync_policy_manager` * Add parameters to set the min/max part size for large file upload/copy methods * Add CopySourceTooBig exception -* Add an option to set a custom file version class to FileVersionFactory +* Add an option to set a custom file version class to `FileVersionFactory` * Add an option for B2Api to turn off hash checking for downloaded files -* Add an option for B2Api to set write buffer size for DownloadedFile.save_to method +* Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method * Add support for multiple profile files for SqliteAccountInfo ### Fixed diff --git a/b2sdk/account_info/abstract.py b/b2sdk/account_info/abstract.py index 086748057..81803e356 100644 --- a/b2sdk/account_info/abstract.py +++ b/b2sdk/account_info/abstract.py @@ -126,6 +126,9 @@ def is_same_account(self, account_id: str, realm: str) -> bool: except exception.MissingAccountData: return False + def is_master_key(self) -> bool: + return self.get_account_id() == self.get_application_key_id() + @abstractmethod def get_account_id(self): """ diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index a710f2046..c8b7376d2 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -180,6 +180,8 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': + if value_dict is None: + return replication_source_dict = value_dict.get('asReplicationSource') as_replication_source = replication_source_dict and ReplicationSourceConfiguration.from_dict( replication_source_dict @@ -215,6 +217,8 @@ def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurati @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfigurationResponse': + if value_dict is None: + return return cls( is_client_authorized_to_read=value_dict['isClientAuthorizedToRead'], value=ReplicationConfiguration.from_dict(value_dict['value']), diff --git a/test/integration/test_download.py b/test/integration/test_download.py index ee9141c5f..a05fe908f 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -22,60 +22,60 @@ from .base import IntegrationTestBase -class TestDownload(IntegrationTestBase): - def test_large_file(self): - bucket = self.create_bucket() - with mock.patch.object( - self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() - ): - download_manager = self.b2_api.services.download_manager - with mock.patch.object( - download_manager, - 'strategies', - new=[ - ParallelDownloader( - min_part_size=self.info.get_absolute_minimum_part_size(), - min_chunk_size=download_manager.MIN_CHUNK_SIZE, - max_chunk_size=download_manager.MAX_CHUNK_SIZE, - thread_pool=download_manager._thread_pool, - ) - ] - ): - - # let's check that small file downloads fail with these settings - bucket.upload_bytes(b'0', 'a_single_zero') - with pytest.raises(ValueError) as exc_info: - with io.BytesIO() as io_: - bucket.download_file_by_name('a_single_zero').save(io_) - assert exc_info.value.args == ('no strategy suitable for download was found!',) - f = self._file_helper(bucket) - assert f.download_version.content_sha1_verified - - def _file_helper(self, bucket, sha1_sum=None): - bytes_to_write = int(self.info.get_absolute_minimum_part_size() * 2.5) - with TempDir() as temp_dir: - temp_dir = pathlib.Path(temp_dir) - source_large_file = pathlib.Path(temp_dir) / 'source_large_file' - with open(source_large_file, 'wb') as large_file: - self.write_zeros(large_file, bytes_to_write) - bucket.upload_local_file( - source_large_file, - 'large_file', - sha1_sum='do_not_verify', - ) - target_large_file = pathlib.Path(temp_dir) / 'target_large_file' - - f = bucket.download_file_by_name('large_file') - f.save_to(target_large_file) - assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file) - return f - - def test_small(self): - bucket = self.create_bucket() - f = self._file_helper(bucket) - assert not f.download_version.content_sha1_verified - - def test_small_unverified(self): - bucket = self.create_bucket() - f = self._file_helper(bucket, sha1_sum='do_not_verify') - assert not f.download_version.content_sha1_verified +#class TestDownload(IntegrationTestBase): +# def test_large_file(self): +# bucket = self.create_bucket() +# with mock.patch.object( +# self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() +# ): +# download_manager = self.b2_api.services.download_manager +# with mock.patch.object( +# download_manager, +# 'strategies', +# new=[ +# ParallelDownloader( +# min_part_size=self.info.get_absolute_minimum_part_size(), +# min_chunk_size=download_manager.MIN_CHUNK_SIZE, +# max_chunk_size=download_manager.MAX_CHUNK_SIZE, +# thread_pool=download_manager._thread_pool, +# ) +# ] +# ): +# +# # let's check that small file downloads fail with these settings +# bucket.upload_bytes(b'0', 'a_single_zero') +# with pytest.raises(ValueError) as exc_info: +# with io.BytesIO() as io_: +# bucket.download_file_by_name('a_single_zero').save(io_) +# assert exc_info.value.args == ('no strategy suitable for download was found!',) +# f = self._file_helper(bucket) +# assert f.download_version.content_sha1_verified +# +# def _file_helper(self, bucket, sha1_sum=None): +# bytes_to_write = int(self.info.get_absolute_minimum_part_size() * 2.5) +# with TempDir() as temp_dir: +# temp_dir = pathlib.Path(temp_dir) +# source_large_file = pathlib.Path(temp_dir) / 'source_large_file' +# with open(source_large_file, 'wb') as large_file: +# self.write_zeros(large_file, bytes_to_write) +# bucket.upload_local_file( +# source_large_file, +# 'large_file', +# sha1_sum='do_not_verify', +# ) +# target_large_file = pathlib.Path(temp_dir) / 'target_large_file' +# +# f = bucket.download_file_by_name('large_file') +# f.save_to(target_large_file) +# assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file) +# return f +# +# def test_small(self): +# bucket = self.create_bucket() +# f = self._file_helper(bucket) +# assert not f.download_version.content_sha1_verified +# +# def test_small_unverified(self): +# bucket = self.create_bucket() +# f = self._file_helper(bucket, sha1_sum='do_not_verify') +# assert not f.download_version.content_sha1_verified From e76b9cac61621177912f502d8eddc81cfa450959 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 18 Apr 2022 14:39:40 +0200 Subject: [PATCH 12/48] Format code --- test/integration/test_download.py | 1 - test/integration/test_raw_api.py | 29 ++++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index a05fe908f..39f2f510c 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -21,7 +21,6 @@ from .helpers import GENERAL_BUCKET_NAME_PREFIX from .base import IntegrationTestBase - #class TestDownload(IntegrationTestBase): # def test_large_file(self): # bucket = self.create_bucket() diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 5699df21a..de2a9d857 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -163,21 +163,24 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): assert 'replicationConfiguration' in replication_source_bucket_dict assert replication_source_bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, - 'value': { - "asReplicationSource": { - "replicationRules": [ + 'value': + { + "asReplicationSource": { - "destinationBucketId": bucket_id, - "fileNamePrefix": "", - "isEnabled": True, - "priority": 1, - "replicationRuleName": "test-rule" + "replicationRules": + [ + { + "destinationBucketId": bucket_id, + "fileNamePrefix": "", + "isEnabled": True, + "priority": 1, + "replicationRuleName": "test-rule" + }, + ], + "sourceApplicationKeyId": key_dict['applicationKeyId'] }, - ], - "sourceApplicationKeyId": key_dict['applicationKeyId'] + "asReplicationDestination": None, }, - "asReplicationDestination": None, - }, } ################## @@ -466,4 +469,4 @@ def _should_delete_bucket(bucket_name): # Is it more than an hour old? bucket_time = int(match.group(1)) now = time.time() - return bucket_time + 3600 <= now \ No newline at end of file + return bucket_time + 3600 <= now From b4490abcd48006081c75ed994175d9403ad94927 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 18 Apr 2022 22:33:34 +0700 Subject: [PATCH 13/48] Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` --- CHANGELOG.md | 1 + b2sdk/raw_api.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcdaba21d..0513ad8db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Add basic replication support to `Bucket` and `FileVersion` * Add `is_master_key()` method to `AbstractAccountInfo` +* Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` ### Fixed * Fix license test on Windows diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index e5b0a3026..d129faf7e 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -47,6 +47,8 @@ 'writeFileRetentions', 'readFileLegalHolds', 'writeFileLegalHolds', + 'readBucketReplications', + 'writeBucketReplications', 'bypassGovernance', 'listFiles', 'readFiles', From cdff74230665e99e253cf1cd0cd72568d609f9bd Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Mon, 18 Apr 2022 19:32:29 +0300 Subject: [PATCH 14/48] Update integration test for replication --- b2sdk/replication/setting.py | 4 +- test/integration/test_raw_api.py | 158 +++++++++++++++++++++++-------- 2 files changed, 122 insertions(+), 40 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index c8b7376d2..7695798b7 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -201,7 +201,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': @dataclass class ReplicationConfigurationResponse: is_client_authorized_to_read: bool - value: ReplicationConfiguration + value: Optional[ReplicationConfiguration] @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurationResponse']: @@ -221,5 +221,5 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationConfigurationResponse': return return cls( is_client_authorized_to_read=value_dict['isClientAuthorizedToRead'], - value=ReplicationConfiguration.from_dict(value_dict['value']), + value=value_dict['value'] and ReplicationConfiguration.from_dict(value_dict['value']), ) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index de2a9d857..2aa68732f 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -19,7 +19,7 @@ from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting -from b2sdk.replication.setting import ReplicationConfiguration, ReplicationSourceConfiguration, ReplicationRule +from b2sdk.replication.setting import ReplicationConfiguration, ReplicationSourceConfiguration, ReplicationRule, ReplicationDestinationConfiguration from b2sdk.file_lock import BucketRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod from b2sdk.raw_api import B2RawHTTPApi, REALM_URLS from b2sdk.utils import hex_sha1_of_stream @@ -138,52 +138,134 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] - # b2_create_bucket/replication-source - print('b2_create_bucket/replication-source') - # in order to test replication, we need to create a second bucket - replication_source_bucket_dict = raw_api.create_bucket( + ################################# + print('b2 / replication') + + # 1) create source key (read permissions) + replication_source_key_dict = raw_api.create_key( api_url, account_auth_token, account_id, - bucket_name + '-rep', - 'allPublic', - is_file_lock_enabled=True, - replication=ReplicationConfiguration( - as_replication_source=ReplicationSourceConfiguration( - replication_rules=[ - ReplicationRule( - destination_bucket_id=bucket_id, - replication_rule_name='test-rule', - ), - ], - source_application_key_id=key_dict['applicationKeyId'], + ['listBuckets', 'listFiles', 'readFiles'], + 'testReplicationSourceKey', + None, + None, + None, + ) + replication_source_key = replication_source_key_dict['applicationKeyId'] + + # 2) create source bucket with replication to destination - existing bucket + try: + # in order to test replication, we need to create a second bucket + replication_source_bucket_dict = raw_api.create_bucket( + api_url, + account_auth_token, + account_id, + bucket_name + '-rep', + 'allPublic', + is_file_lock_enabled=True, + replication=ReplicationConfiguration( + as_replication_source=ReplicationSourceConfiguration( + replication_rules=[ + ReplicationRule( + destination_bucket_id=bucket_id, + replication_rule_name='test-rule', + ), + ], + source_application_key_id=replication_source_key, + ), ), - ), + ) + assert 'replicationConfiguration' in replication_source_bucket_dict + assert replication_source_bucket_dict['replicationConfiguration'] == { + 'isClientAuthorizedToRead': True, + 'value': + { + "asReplicationSource": + { + "replicationRules": + [ + { + "destinationBucketId": bucket_id, + "fileNamePrefix": "", + "isEnabled": True, + "priority": 1, + "replicationRuleName": "test-rule" + }, + ], + "sourceApplicationKeyId": replication_source_key, + }, + "asReplicationDestination": None, + }, + } + finally: + raw_api.delete_key(api_url, account_auth_token, replication_source_key) + + # 3) create destination key (write permissions) + replication_destination_key_dict = raw_api.create_key( + api_url, + account_auth_token, + account_id, + ['listBuckets', 'listFiles', 'writeFiles'], + 'testReplicationDestinationKey', + None, + None, + None, ) - assert 'replicationConfiguration' in replication_source_bucket_dict - assert replication_source_bucket_dict['replicationConfiguration'] == { - 'isClientAuthorizedToRead': True, - 'value': - { - "asReplicationSource": - { - "replicationRules": - [ + replication_destination_key = replication_destination_key_dict['applicationKeyId'] + + # 4) update destination bucket to receive updates + try: + bucket_dict = raw_api.update_bucket( + api_url, + account_auth_token, + account_id, + bucket_id, + 'allPublic', + replication=ReplicationConfiguration( + as_replication_destination=ReplicationDestinationConfiguration( + source_to_destination_key_mapping={ + replication_source_key: replication_destination_key, + }, + ), + ), + ) + assert bucket_dict['replicationConfiguration'] == { + 'isClientAuthorizedToRead': True, + 'value': + { + 'asReplicationDestination': + { + 'sourceToDestinationKeyMapping': { - "destinationBucketId": bucket_id, - "fileNamePrefix": "", - "isEnabled": True, - "priority": 1, - "replicationRuleName": "test-rule" + replication_source_key: replication_destination_key, }, - ], - "sourceApplicationKeyId": key_dict['applicationKeyId'] - }, - "asReplicationDestination": None, - }, + }, + 'asReplicationSource': None, + }, + } + finally: + raw_api.delete_key( + api_url, + account_auth_token, + replication_destination_key_dict['applicationKeyId'], + ) + + # 5) cleanup: disable replication for destination + bucket_dict = raw_api.update_bucket( + api_url, + account_auth_token, + account_id, + bucket_id, + 'allPublic', + replication=ReplicationConfiguration(), + ) + assert bucket_dict['replicationConfiguration'] == { + 'isClientAuthorizedToRead': True, + 'value': None, } - ################## + ################# print('b2_update_bucket') sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, From d49d723b9ae339ac8d1ac69b236243d6b21150bb Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 19 Apr 2022 03:48:30 +0300 Subject: [PATCH 15/48] Fix Bucket's as_dict() --- b2sdk/bucket.py | 50 ++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index b51058fc9..c77120257 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -955,6 +955,7 @@ def as_dict(self): result['defaultServerSideEncryption'] = self.default_server_side_encryption.as_dict() result['isFileLockEnabled'] = self.is_file_lock_enabled result['defaultRetention'] = self.default_retention.as_dict() + result['replication'] = self.replication and self.replication.as_dict() return result @@ -1012,29 +1013,32 @@ def from_api_bucket_dict(cls, api, bucket_dict): } }, "replicationConfiguration": { - "asReplicationSource": { - "replicationRules": [ - { - "destinationBucketId": "c5f35d53a90a7ea284fb0719", - "fileNamePrefix": "", - "isEnabled": true, - "priority": 1, - "replicationRuleName": "replication-us-west" - }, - { - "destinationBucketId": "55f34d53a96a7ea284fb0719", - "fileNamePrefix": "", - "isEnabled": true, - "priority": 2, - "replicationRuleName": "replication-us-west-2" + "clientIsAllowedToRead": true, + "value": { + "asReplicationSource": { + "replicationRules": [ + { + "destinationBucketId": "c5f35d53a90a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 1, + "replicationRuleName": "replication-us-west" + }, + { + "destinationBucketId": "55f34d53a96a7ea284fb0719", + "fileNamePrefix": "", + "isEnabled": true, + "priority": 2, + "replicationRuleName": "replication-us-west-2" + } + ], + "sourceApplicationKeyId": "10053d55ae26b790000000006" + }, + "asReplicationDestination": { + "sourceToDestinationKeyMapping": { + "10053d55ae26b790000000045": "10053d55ae26b790000000004", + "10053d55ae26b790000000046": "10053d55ae26b790030000004" } - ], - "sourceApplicationKeyId": "10053d55ae26b790000000006" - }, - "asReplicationDestination": { - "sourceToDestinationKeyMapping": { - "10053d55ae26b790000000045": "10053d55ae26b790000000004", - "10053d55ae26b790000000046": "10053d55ae26b790030000004" } } } @@ -1077,5 +1081,5 @@ def from_api_bucket_dict(cls, api, bucket_dict): default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, - replication, + replication.value, ) From e023712026b93e20794f3c44f57dffadb413af3a Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 19 Apr 2022 03:49:17 +0300 Subject: [PATCH 16/48] Add ReplicationStatus to DownloadedFile --- b2sdk/transfer/inbound/download_manager.py | 3 +++ b2sdk/transfer/inbound/downloaded_file.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/b2sdk/transfer/inbound/download_manager.py b/b2sdk/transfer/inbound/download_manager.py index 008ce6a47..ecaeb9db4 100644 --- a/b2sdk/transfer/inbound/download_manager.py +++ b/b2sdk/transfer/inbound/download_manager.py @@ -106,4 +106,7 @@ def download_file_from_url( progress_listener=progress_listener, write_buffer_size=self.write_buffer_size, check_hash=self.check_hash, + replication_status=DownloadedFile.get_replication_status_from_headers( + response.headers, + ), ) diff --git a/b2sdk/transfer/inbound/downloaded_file.py b/b2sdk/transfer/inbound/downloaded_file.py index 07751166a..ec036e846 100644 --- a/b2sdk/transfer/inbound/downloaded_file.py +++ b/b2sdk/transfer/inbound/downloaded_file.py @@ -15,6 +15,7 @@ from requests.models import Response from ...encryption.setting import EncryptionSetting +from ...replication.types import ReplicationStatus from ...file_version import DownloadVersion from ...progress import AbstractProgressListener from ...stream.progress import WritingStreamWithProgress @@ -96,6 +97,7 @@ def __init__( progress_listener: AbstractProgressListener, write_buffer_size=None, check_hash=True, + replication_status: Optional[ReplicationStatus] = None, ): self.download_version = download_version self.download_manager = download_manager @@ -106,6 +108,7 @@ def __init__( self.download_strategy = None self.write_buffer_size = write_buffer_size self.check_hash = check_hash + self.replication_status = replication_status def _validate_download(self, bytes_read, actual_sha1): if self.range_ is None: @@ -172,3 +175,7 @@ def save_to(self, path_, mode='wb+', allow_seeking=True): buffering=self.write_buffer_size, ) as file: self.save(file, allow_seeking=allow_seeking) + + @classmethod + def get_replication_status_from_headers(headers: dict) -> Optional[ReplicationStatus]: + return headers.get('X-Bz-Replication-Status', None) From 1160cbf9cef369c0418cd295f609cc7c02499b2c Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 19 Apr 2022 11:23:03 +0200 Subject: [PATCH 17/48] Try to fix unit tests --- b2sdk/bucket.py | 2 +- b2sdk/transfer/inbound/downloaded_file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index c77120257..092b4dc6f 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -1081,5 +1081,5 @@ def from_api_bucket_dict(cls, api, bucket_dict): default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, - replication.value, + replication and replication.value, ) diff --git a/b2sdk/transfer/inbound/downloaded_file.py b/b2sdk/transfer/inbound/downloaded_file.py index ec036e846..3d510f354 100644 --- a/b2sdk/transfer/inbound/downloaded_file.py +++ b/b2sdk/transfer/inbound/downloaded_file.py @@ -177,5 +177,5 @@ def save_to(self, path_, mode='wb+', allow_seeking=True): self.save(file, allow_seeking=allow_seeking) @classmethod - def get_replication_status_from_headers(headers: dict) -> Optional[ReplicationStatus]: + def get_replication_status_from_headers(cls, headers: dict) -> Optional[ReplicationStatus]: return headers.get('X-Bz-Replication-Status', None) From 0e40d3c42cf61801e6bae3ffb703035cfde57002 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 19 Apr 2022 17:01:33 +0300 Subject: [PATCH 18/48] Move ReplicationStatus to DownloadFileVersion; fix all tests --- b2sdk/file_version.py | 15 ++++++++++----- b2sdk/replication/types.py | 6 ++++++ b2sdk/transfer/inbound/download_manager.py | 3 --- b2sdk/transfer/inbound/downloaded_file.py | 7 ------- test/unit/bucket/test_bucket.py | 5 +---- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index ab174c08b..61cd7d456 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -45,6 +45,7 @@ class BaseFileVersion: 'legal_hold', 'file_retention', 'mod_time_millis', + 'replication_status', ] def __init__( @@ -60,6 +61,7 @@ def __init__( server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, + replication_status: Optional[ReplicationStatus] = None, ): self.api = api self.id_ = id_ @@ -72,6 +74,7 @@ def __init__( self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold + self.replication_status = replication_status if SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS]) @@ -111,6 +114,7 @@ def _get_args_for_clone(self): 'server_side_encryption': self.server_side_encryption, 'file_retention': self.file_retention, 'legal_hold': self.legal_hold, + 'replication_status': self.replication_status, } # yapf: disable def as_dict(self): @@ -134,6 +138,7 @@ def as_dict(self): result['contentSha1'] = self._encode_content_sha1( self.content_sha1, self.content_sha1_verified ) + result['replicationStatus'] = self.replication_status and self.replication_status.value return result @@ -192,7 +197,6 @@ class FileVersion(BaseFileVersion): 'bucket_id', 'content_md5', 'action', - 'replication_status', ] def __init__( @@ -212,13 +216,12 @@ def __init__( server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, - replication_status: ReplicationStatus = None, + replication_status: Optional[ReplicationStatus] = None, ): self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.action = action - self.replication_status = replication_status super().__init__( api=api, @@ -232,6 +235,7 @@ def __init__( server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, + replication_status=replication_status, ) def _get_args_for_clone(self): @@ -242,7 +246,6 @@ def _get_args_for_clone(self): 'bucket_id': self.bucket_id, 'action': self.action, 'content_md5': self.content_md5, - 'replication_status': self.replication_status, } ) return args @@ -251,7 +254,6 @@ def as_dict(self): result = super().as_dict() result['accountId'] = self.account_id result['bucketId'] = self.bucket_id - result['replicationStatus'] = self.replication_status and self.replication_status.value if self.action is not None: result['action'] = self.action @@ -315,6 +317,7 @@ def __init__( content_encoding: Optional[str], file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, + replication_status: Optional[ReplicationStatus] = None, ): self.range_ = range_ self.content_disposition = content_disposition @@ -336,6 +339,7 @@ def __init__( server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, + replication_status=replication_status, ) def _get_args_for_clone(self): @@ -497,6 +501,7 @@ def from_response_headers(self, headers): content_encoding=headers.get('Content-Encoding'), file_retention=FileRetentionSetting.from_response_headers(headers), legal_hold=LegalHold.from_response_headers(headers), + replication_status=ReplicationStatus.from_response_headers(headers), ) diff --git a/b2sdk/replication/types.py b/b2sdk/replication/types.py index 687f88f54..235dc5c40 100644 --- a/b2sdk/replication/types.py +++ b/b2sdk/replication/types.py @@ -9,6 +9,7 @@ ###################################################################### from enum import Enum, unique +from typing import Optional @unique @@ -17,3 +18,8 @@ class ReplicationStatus(Enum): COMPLETED = 'COMPLETED' FAILED = 'FAILED' REPLICA = 'REPLICA' + + @classmethod + def from_response_headers(cls, headers: dict) -> Optional['ReplicationStatus']: + value = headers.get('X-Bz-Replication-Status', None) + return value and cls[value] diff --git a/b2sdk/transfer/inbound/download_manager.py b/b2sdk/transfer/inbound/download_manager.py index ecaeb9db4..008ce6a47 100644 --- a/b2sdk/transfer/inbound/download_manager.py +++ b/b2sdk/transfer/inbound/download_manager.py @@ -106,7 +106,4 @@ def download_file_from_url( progress_listener=progress_listener, write_buffer_size=self.write_buffer_size, check_hash=self.check_hash, - replication_status=DownloadedFile.get_replication_status_from_headers( - response.headers, - ), ) diff --git a/b2sdk/transfer/inbound/downloaded_file.py b/b2sdk/transfer/inbound/downloaded_file.py index 3d510f354..07751166a 100644 --- a/b2sdk/transfer/inbound/downloaded_file.py +++ b/b2sdk/transfer/inbound/downloaded_file.py @@ -15,7 +15,6 @@ from requests.models import Response from ...encryption.setting import EncryptionSetting -from ...replication.types import ReplicationStatus from ...file_version import DownloadVersion from ...progress import AbstractProgressListener from ...stream.progress import WritingStreamWithProgress @@ -97,7 +96,6 @@ def __init__( progress_listener: AbstractProgressListener, write_buffer_size=None, check_hash=True, - replication_status: Optional[ReplicationStatus] = None, ): self.download_version = download_version self.download_manager = download_manager @@ -108,7 +106,6 @@ def __init__( self.download_strategy = None self.write_buffer_size = write_buffer_size self.check_hash = check_hash - self.replication_status = replication_status def _validate_download(self, bytes_read, actual_sha1): if self.range_ is None: @@ -175,7 +172,3 @@ def save_to(self, path_, mode='wb+', allow_seeking=True): buffering=self.write_buffer_size, ) as file: self.save(file, allow_seeking=allow_seeking) - - @classmethod - def get_replication_status_from_headers(cls, headers: dict) -> Optional[ReplicationStatus]: - return headers.get('X-Bz-Replication-Status', None) diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index fa695495d..5712f564d 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -1023,10 +1023,7 @@ def test_update(self): 'options_set': set(), 'default_server_side_encryption': SSE_B2_AES, 'default_retention': BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=7)), - 'replication': ReplicationConfigurationResponse( - is_client_authorized_to_read=True, - value=REPLICATION, - ), + 'replication': REPLICATION, } for attr_name, attr_value in assertions_mapping.items(): self.assertEqual(attr_value, getattr(result, attr_name), attr_name) From 13d8df78036a30a858a555a68ebd63667b86a07c Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 19 Apr 2022 21:51:12 +0300 Subject: [PATCH 19/48] Add integration test for ReplicationStatus --- b2sdk/file_version.py | 6 +++--- b2sdk/replication/types.py | 2 +- test/integration/test_raw_api.py | 35 ++++++++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 61cd7d456..bcf764c2d 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -380,7 +380,7 @@ def from_api_response(self, file_version_dict, force_action=None): "fileName": "randomdata", "size": 0, "uploadTimestamp": 1451444477000, - "replicationStatus": "PENDING" + "replicationStatus": "pending" } or this: @@ -397,7 +397,7 @@ def from_api_response(self, file_version_dict, force_action=None): "fileInfo": {}, "fileName": "randomdata", "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, - "replicationStatus": "COMPLETED" + "replicationStatus": "completed" } into a :py:class:`b2sdk.v2.FileVersion` object. @@ -425,7 +425,7 @@ def from_api_response(self, file_version_dict, force_action=None): legal_hold = LegalHold.from_file_version_dict(file_version_dict) replication_status_value = file_version_dict.get('replicationStatus') - replication_status = replication_status_value and ReplicationStatus[replication_status_value] + replication_status = replication_status_value and ReplicationStatus[replication_status_value.upper()] return self.FILE_VERSION_CLASS( self.api, diff --git a/b2sdk/replication/types.py b/b2sdk/replication/types.py index 235dc5c40..bf11217c4 100644 --- a/b2sdk/replication/types.py +++ b/b2sdk/replication/types.py @@ -22,4 +22,4 @@ class ReplicationStatus(Enum): @classmethod def from_response_headers(cls, headers: dict) -> Optional['ReplicationStatus']: value = headers.get('X-Bz-Replication-Status', None) - return value and cls[value] + return value and cls[value.upper()] diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 2aa68732f..04b5a8c4d 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -20,6 +20,7 @@ from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting from b2sdk.replication.setting import ReplicationConfiguration, ReplicationSourceConfiguration, ReplicationRule, ReplicationDestinationConfiguration +from b2sdk.replication.types import ReplicationStatus from b2sdk.file_lock import BucketRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod from b2sdk.raw_api import B2RawHTTPApi, REALM_URLS from b2sdk.utils import hex_sha1_of_stream @@ -157,11 +158,14 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): # 2) create source bucket with replication to destination - existing bucket try: # in order to test replication, we need to create a second bucket + replication_source_bucket_name = 'test-raw-api-%s-%d-%d' % ( + account_id, int(time.time()), random.randint(1000, 9999) + ) replication_source_bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, - bucket_name + '-rep', + replication_source_bucket_name, 'allPublic', is_file_lock_enabled=True, replication=ReplicationConfiguration( @@ -198,10 +202,29 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): "asReplicationDestination": None, }, } + + # # 3) upload test file and check replication status + upload_url_dict = raw_api.get_upload_url( + api_url, account_auth_token, replication_source_bucket_dict['bucketId'], + ) + file_contents = b'hello world' + file_dict = raw_api.upload_file( + upload_url_dict['uploadUrl'], + upload_url_dict['authorizationToken'], + 'test.txt', + len(file_contents), + 'text/plain', + hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)), + {'color': 'blue'}, + io.BytesIO(file_contents), + ) + + assert ReplicationStatus[file_dict['replicationStatus'].upper()] + finally: raw_api.delete_key(api_url, account_auth_token, replication_source_key) - # 3) create destination key (write permissions) + # 4) create destination key (write permissions) replication_destination_key_dict = raw_api.create_key( api_url, account_auth_token, @@ -214,7 +237,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): ) replication_destination_key = replication_destination_key_dict['applicationKeyId'] - # 4) update destination bucket to receive updates + # 5) update destination bucket to receive updates try: bucket_dict = raw_api.update_bucket( api_url, @@ -251,7 +274,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): replication_destination_key_dict['applicationKeyId'], ) - # 5) cleanup: disable replication for destination + # 6) cleanup: disable replication for destination and remove source bucket_dict = raw_api.update_bucket( api_url, account_auth_token, @@ -265,6 +288,10 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): 'value': None, } + _clean_and_delete_bucket( + raw_api, api_url, account_auth_token, account_id, replication_source_bucket_dict['bucketId'], + ) + ################# print('b2_update_bucket') sse_b2_aes = EncryptionSetting( From c2576c8af0d664c330a2e55629a9a5a6b9706c76 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 19 Apr 2022 21:59:12 +0300 Subject: [PATCH 20/48] Lint fix --- b2sdk/file_version.py | 3 ++- test/integration/test_raw_api.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index bcf764c2d..4934cb9da 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -425,7 +425,8 @@ def from_api_response(self, file_version_dict, force_action=None): legal_hold = LegalHold.from_file_version_dict(file_version_dict) replication_status_value = file_version_dict.get('replicationStatus') - replication_status = replication_status_value and ReplicationStatus[replication_status_value.upper()] + replication_status = replication_status_value and ReplicationStatus[ + replication_status_value.upper()] return self.FILE_VERSION_CLASS( self.api, diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 04b5a8c4d..961dd9e26 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -205,7 +205,9 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): # # 3) upload test file and check replication status upload_url_dict = raw_api.get_upload_url( - api_url, account_auth_token, replication_source_bucket_dict['bucketId'], + api_url, + account_auth_token, + replication_source_bucket_dict['bucketId'], ) file_contents = b'hello world' file_dict = raw_api.upload_file( @@ -289,7 +291,11 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): } _clean_and_delete_bucket( - raw_api, api_url, account_auth_token, account_id, replication_source_bucket_dict['bucketId'], + raw_api, + api_url, + account_auth_token, + account_id, + replication_source_bucket_dict['bucketId'], ) ################# From b02de3ac718be38fff61b2d0fbac2030a72af20b Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 19 Apr 2022 03:04:43 +0700 Subject: [PATCH 21/48] Add ReplicationSetupHelper, rename a couple of replication fields --- b2sdk/api.py | 13 ++ b2sdk/raw_simulator.py | 2 +- b2sdk/replication/setting.py | 20 +- b2sdk/replication/setup.py | 303 ++++++++++++++++++++++++++++ test/integration/test_raw_api.py | 4 +- test/unit/bucket/test_bucket.py | 11 +- test/unit/v_all/test_replication.py | 60 ++++++ 7 files changed, 397 insertions(+), 16 deletions(-) create mode 100644 b2sdk/replication/setup.py create mode 100644 test/unit/v_all/test_replication.py diff --git a/b2sdk/api.py b/b2sdk/api.py index 72f30fcef..f043f9b9e 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -530,6 +530,19 @@ def list_keys(self, start_application_key_id: Optional[str] = None return start_application_key_id = next_application_key_id + def get_key(self, key_id: str) -> Optional[ApplicationKey]: + """ + Gets information about a single key: it's capabilities, prefix, name etc + + Returns `None` if the key does not exist. + + Raises an exception if profile is not permitted to list keys. + """ + try: + return self.list_keys(start_application_key_id=key_id)[0] + except IndexError: + return None + # other def get_file_info(self, file_id: str) -> FileVersion: """ diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 7e2521c85..9e5211df6 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1711,7 +1711,7 @@ def update_bucket( default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): - assert bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption + assert bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption or replication bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeBuckets') return bucket._update_bucket( diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 7695798b7..16279b2bc 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -23,7 +23,7 @@ class ReplicationRule: """ destination_bucket_id: str - replication_rule_name: str + name: str file_name_prefix: str = '' is_enabled: bool = True priority: int = 1 @@ -40,8 +40,8 @@ def __post_init__(self): # TODO: file_name_prefix validation - if not self.REPLICATION_RULE_REGEX.match(self.replication_rule_name): - raise ValueError('replication_rule_name is invalid') + if not self.REPLICATION_RULE_REGEX.match(self.name): + raise ValueError('replication rule name is invalid') def as_dict(self) -> dict: return { @@ -49,7 +49,7 @@ def as_dict(self) -> dict: 'fileNamePrefix': self.file_name_prefix, 'isEnabled': self.is_enabled, 'priority': self.priority, - 'replicationRuleName': self.replication_rule_name, + 'replicationRuleName': self.name, } @classmethod @@ -59,7 +59,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationRule': file_name_prefix=value_dict['fileNamePrefix'], is_enabled=value_dict['isEnabled'], priority=value_dict['priority'], - replication_rule_name=value_dict['replicationRuleName'], + name=value_dict['replicationRuleName'], ) @@ -69,26 +69,26 @@ class ReplicationSourceConfiguration: Hold information about bucket being a replication source """ - replication_rules: List[ReplicationRule] + rules: List[ReplicationRule] source_application_key_id: str def __post_init__(self): - if not self.replication_rules: - raise ValueError("replication_rules must not be empty") + if not self.rules: + raise ValueError("rules must not be empty") if not self.source_application_key_id: raise ValueError("source_application_key_id must not be empty") def as_dict(self) -> dict: return { - "replicationRules": [rule.as_dict() for rule in self.replication_rules], + "replicationRules": [rule.as_dict() for rule in self.rules], "sourceApplicationKeyId": self.source_application_key_id, } @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationSourceConfiguration': return cls( - replication_rules=[ + rules=[ ReplicationRule.from_dict(rule_dict) for rule_dict in value_dict['replicationRules'] ], source_application_key_id=value_dict['sourceApplicationKeyId'], diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py new file mode 100644 index 000000000..c75f22cc8 --- /dev/null +++ b/b2sdk/replication/setup.py @@ -0,0 +1,303 @@ +###################################################################### +# +# File: b2sdk/replication/setting.py +# +# Copyright 2022 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +# b2 replication-setup [--create-keys (both|source|destination|auto)] [--widen-source-key-mode (fail|auto)] [--profile profileName] [--priority int|auto]--destination-profile destinationProfileName sourceBucketPath destinationBucketPath [rrName] +# b2 replication-debug [--profile profileName] [--destination-profile destinationProfileName] bucketPath +# b2 replication-status [--profile profileName] [--destination-profile destinationProfileName] [sourceBucketPath] [destinationBucketPath] +# b2 replication-pause [--profile profileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] +# b2 replication-unpause [--profile profileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] +# b2 replication-scan [--profile profileName] [--destination-profile destinationProfileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] + +from typing import ClassVar, List, Tuple +from enum import Enum, auto, unique +import logging + +from b2sdk.api import B2Api +from b2sdk.application_key import ApplicationKey +from b2sdk.bucket import Bucket +from b2sdk.replication.setting import ReplicationConfiguration, ReplicationDestinationConfiguration, ReplicationRule, ReplicationSourceConfiguration + +logger = logging.getLogger(__name__) + + +@unique +class ReplicationSetupHelperKeyMode(Enum): + NONE = auto() + AUTO = auto() + SOURCE = auto() + DESTINATION = auto() + BOTH = auto() + + +class ReplicationSetupHelper: + """ class with various methods that help with repliction management """ + DEFAULT_KEY_MODE: ClassVar[ + ReplicationSetupHelperKeyMode + ] = ReplicationSetupHelperKeyMode.AUTO #: default key mode for setup signatures + PRIORITY_OFFSET: ClassVar[int] = 10 #: how far to to put the new rule from the existing rules + DEFAULT_PRIORITY: ClassVar[int] = 128 #: what priority to set if there are no preexisting rules + MAX_PRIORITY: ClassVar[int] = 255 #: maximum allowed priority of a replication rule + DEFAULT_SOURCE_CAPABILITIES = 'readFiles' + + def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): + assert source_b2api is not None or destination_b2api is not None + self.source_b2api = source_b2api + self.destination_b2api = destination_b2api + + def setup_both( + self, + source_bucket_path: str, + destination_bucket: Bucket, + name: str = None, #: name for the new replication rule + priority: int = None, #: priority for the new replication rule + key_mode: ReplicationSetupHelperKeyMode = DEFAULT_KEY_MODE, + widen_source_key: bool = True, + ): + source_key: ApplicationKey = self.setup_source( + source_bucket_path, + destination_bucket.id_, + name, + priority, + key_mode, + widen_source_key, + ) + assert source_key + # TODO + #self.setup_destination(source_key.id_, destination_bucket, key_mode) + + def setup_destination( + self, + api: B2Api, + source_key_id: str, + destination_bucket: Bucket, + key_mode: ReplicationSetupHelperKeyMode = DEFAULT_KEY_MODE, + ): + try: + source_configuration = destination_bucket.replication.as_replication_source + except (NameError, AttributeError): + source_configuration = None + try: + destination_configuration = destination_bucket.replication.as_replication_destination + except (NameError, AttributeError): + destination_configuration = ReplicationDestinationConfiguration({}) + + current_destination_key = destination_configuration.source_to_destination_key_mapping.get( + source_key_id + ) + + destination_key = self._get_destination_key(current_destination_key) + destination_configuration.source_to_destination_key_mapping[source_key_id] = destination_key + new_replication_configuration = ReplicationConfiguration( + source_configuration, + destination_configuration, + ) + return new_replication_configuration + + def _get_destination_key(self, current_destination_key) -> ApplicationKey: + # name=source_bucket.name[:91] + '-replisrc', + do_create_key = False + if current_destination_key is None: + do_create_key = True + if current_destination_key.prefix: + pass + # TODO + if not do_create_key: + return current_destination_key + + def setup_source( + self, source_bucket_path, destination_bucket_id, name, priority, key_mode, widen_source_key + ) -> ApplicationKey: + source_bucket_name, prefix = self._partion_bucket_path(source_bucket_path) + source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh + + try: + current_source_rrs = source_bucket.replication.as_replication_source.rules + except (NameError, AttributeError): + current_source_rrs = [] + try: + destination_configuration = source_bucket.replication.as_replication_destination + except (NameError, AttributeError): + destination_configuration = None + + source_key_id = self._get_source_key( + source_bucket, + prefix, + key_mode, + widen_source_key, + source_bucket.replication, + current_source_rrs, + ) + priority = self._get_priority_for_new_rule( + priority, + current_source_rrs, + ) + + new_rr = ReplicationRule( + name=name, + priority=priority, + destination_bucket_id=destination_bucket_id, + file_name_prefix=prefix, + ) + new_replication_configuration = ReplicationConfiguration( + ReplicationSourceConfiguration( + source_application_key_id=source_key_id, + rules=current_source_rrs + [new_rr], + ), + destination_configuration, + ) + source_bucket.update( + if_revision_is=source_bucket.revision, + replication=new_replication_configuration, + ) + return source_key_id + + @classmethod + def _get_source_key( + cls, + source_bucket, + prefix, + key_mode, + widen_source_key, + current_replication_configuration: ReplicationConfiguration, + current_source_rrs, + ) -> str: + assert widen_source_key # TODO + api = source_bucket.api + + force_source_key_creation: bool = key_mode in ( + ReplicationSetupHelperKeyMode.SOURCE, + ReplicationSetupHelperKeyMode.BOTH, + #ReplicationSetupHelperKeyMode.AUTO, + ) + #assert force_source_key_creation # TODO + #create_source_key_if_needed: bool = key_mode is ReplicationSetupHelperKeyMode.LAZY + + do_create_key = False + new_prefix = cls._get_narrowest_common_prefix( + [rr.path for rr in current_source_rrs] + [prefix] + ) + if new_prefix != prefix: + logger.debug( + 'forced key creation due to widened key prefix from %s to %s', + prefix, + new_prefix, + ) + prefix = new_prefix + do_create_key = True + + if force_source_key_creation: + logger.debug( + 'forced key creation because key_mode is %s', + ReplicationSetupHelperKeyMode(key_mode).name, + ) + do_create_key = True + + if not do_create_key: + if not current_replication_configuration or not current_replication_configuration.as_replication_source: + do_create_key = True + else: + current_source_key = api.get_key( + current_replication_configuration.as_replication_source.source_key_id + ) + if current_source_key is None: + do_create_key = True + logger.debug( + 'will create a new source key because current key %s has been deleted', + current_replication_configuration.source_key_id, + ) + else: + current_capabilities = current_source_key.get_capabilities() + if not prefix.startswith(current_source_key.prefix): + do_create_key = True + logger.debug( + 'will create a new source key because %s installed so far does not encompass current needs with its prefix', + current_source_key.id_, + ) + elif 'readFiles' not in current_capabilities: # TODO: more permissions + do_create_key = True + logger.debug( + 'will create a new source key because %s installed so far does not have enough permissions for replication source', + current_source_key.id_, + ) + else: + return current_source_key + + #if not prefix.startswith(current_key.prefix): + # do_create_key = True + #else: + # new_key = current_key + + new_key = cls._create_source_key( + name=source_bucket.name[:91] + '-replisrc', + api=api, + bucket_id=source_bucket.id_, + prefix=prefix, + ) + return new_key + + @classmethod + def _create_source_key( + cls, + name: str, + api: B2Api, + bucket_id: str, + prefix: str, + ) -> ApplicationKey: + capabilities = cls.DEFAULT_SOURCE_CAPABILITIES + return cls._create_key(name, api, bucket_id, prefix, capabilities) + + @classmethod + def _create_destination_key( + cls, + name: str, + api: B2Api, + bucket_id: str, + prefix: str, + ) -> ApplicationKey: + capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES + return cls._create_key(name, api, bucket_id, prefix, capabilities) + + @classmethod + def _create_key( + cls, + name: str, + api: B2Api, + bucket_id: str, + prefix: str, + capabilities, + ) -> ApplicationKey: + return api.create_key( + capabilities=capabilities, + key_name=name, + bucket_id=bucket_id, + name_prefix=prefix, + ) + + @classmethod + def _get_narrowest_common_prefix(cls, widen_to: List[str]) -> str: + for path in widen_to: + pass # TODO + return '' + + @classmethod + def _get_priority_for_new_rule(cls, priority, current_source_rrs): + # if there is no priority hint, look into current rules to determine the last priority and add a constant to it + if priority is not None: + return priority + if current_source_rrs: + # TODO: maybe handle a case where the existing rrs need to have their priorities decreased to make space + existing_priority = max(rr.priority for rr in current_source_rrs) + return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY) + return cls.DEFAULT_PRIORITY + + @classmethod + def _partion_bucket_path(cls, bucket_path: str) -> Tuple[str, str]: + bucket_name, _, path = bucket_path.partition('/') + return bucket_name, path diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 961dd9e26..974af7744 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -170,10 +170,10 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): is_file_lock_enabled=True, replication=ReplicationConfiguration( as_replication_source=ReplicationSourceConfiguration( - replication_rules=[ + rules=[ ReplicationRule( destination_bucket_id=bucket_id, - replication_rule_name='test-rule', + name='test-rule', ), ], source_application_key_id=replication_source_key, diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 5712f564d..81e3da0b5 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -91,14 +91,14 @@ ) REPLICATION = ReplicationConfiguration( as_replication_source=ReplicationSourceConfiguration( - replication_rules=[ + rules=[ ReplicationRule( destination_bucket_id='c5f35d53a90a7ea284fb0719', - replication_rule_name='replication-us-west', + name='replication-us-west', ), ReplicationRule( destination_bucket_id='55f34d53a96a7ea284fb0719', - replication_rule_name='replication-us-west-2', + name='replication-us-west-2', file_name_prefix='replica/', is_enabled=False, priority=255, @@ -1026,6 +1026,11 @@ def test_update(self): 'replication': REPLICATION, } for attr_name, attr_value in assertions_mapping.items(): + self.maxDiff = None + print('---', attr_name, '---') + print(attr_value) + print('?=?') + print(getattr(result, attr_name)) self.assertEqual(attr_value, getattr(result, attr_name), attr_name) def test_update_if_revision_is(self): diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py new file mode 100644 index 000000000..bda001da7 --- /dev/null +++ b/test/unit/v_all/test_replication.py @@ -0,0 +1,60 @@ +###################################################################### +# +# File: test/unit/v_all/test_api.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import pytest + +from apiver_deps import B2Api +from apiver_deps import B2HttpApiConfig +from apiver_deps import Bucket +from apiver_deps import InMemoryCache +from apiver_deps import InMemoryAccountInfo +from apiver_deps import RawSimulator +from ..test_base import TestBase + +from b2sdk.replication.setup import ReplicationSetupHelper + + +class TestReplication(TestBase): + def setUp(self): + self.account_info = InMemoryAccountInfo() + self.cache = InMemoryCache() + self.api = B2Api( + self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) + ) + self.raw_api = self.api.session.raw_api + self.application_key_id, self.master_key = self.raw_api.create_account() + + def _authorize_account(self): + self.api.authorize_account('production', self.application_key_id, self.master_key) + + @pytest.mark.apiver(from_ver=2) + def test_get_bucket_by_id_v2(self): + self._authorize_account() + #with pytest.raises(BucketIdNotFound): + # self.api.get_bucket_by_id("this id doesn't even exist") + created_bucket = self.api.create_bucket('bucket1', 'allPrivate') + destination_bucket = self.api.create_bucket('bucket2', 'allPrivate') + #read_bucket = self.api.get_bucket_by_id(created_bucket.id_) + #assert created_bucket.id_ == read_bucket.id_ + #self.cache.save_bucket(Bucket(api=self.api, name='bucket_name', id_='bucket_id')) + #read_bucket = self.api.get_bucket_by_id('bucket_id') + #assert read_bucket.name == 'bucket_name' + rsh = ReplicationSetupHelper( + source_b2api=self.api, + destination_b2api=self.api, + ) + rsh.setup_both( + source_bucket_path="bucket1", + destination_bucket=destination_bucket, + name='aa', + #priority=None, + #key_mode, + #widen_source_key, + ) From a39971055e6d23ad9fabf189f70f9d34df37c2de Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Mon, 18 Apr 2022 13:26:14 +0200 Subject: [PATCH 22/48] Prepare replication prerelease --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0513ad8db..4a3042dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.16.0-alpha.1] - 2022-04-18 + ### Added * Add basic replication support to `Bucket` and `FileVersion` * Add `is_master_key()` method to `AbstractAccountInfo` From 1ba680ce96bd727ee000e45214024b5ab9af457a Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 20 Apr 2022 00:39:43 +0200 Subject: [PATCH 23/48] 1.16.0-alpha.1 changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a3042dbe..0e1113262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.16.0-alpha.1] - 2022-04-18 +## [1.16.0-alpha.1] - 2022-04-20 + +This is an alpha release. A preview, not for production use. It allows for basic +usage of B2 replication feature (currently in closed beta). The main reason of +releasing it early is to pair it with alpha b2 cli. Expect substantial amount +of work on sdk interface: +* The interface of `Bucket.replication` WILL change +* The interface of `FileVersion.replication_status` MIGHT change +* The interface of `FileVersionDownload` MIGHT change ### Added * Add basic replication support to `Bucket` and `FileVersion` From 0f6bb97427e9e4c0ee072286aab74bf8c3e0b06f Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 20 Apr 2022 00:48:19 +0200 Subject: [PATCH 24/48] Fix changelog header --- b2sdk/replication/setup.py | 2 +- test/unit/v_all/test_replication.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index c75f22cc8..2463b60ab 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/replication/setting.py +# File: b2sdk/replication/setup.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index bda001da7..56841ace9 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: test/unit/v_all/test_api.py +# File: test/unit/v_all/test_replication.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # From 9feaac73907408922d5bec05c2d6d118a1757e94 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 20 Apr 2022 17:15:19 +0300 Subject: [PATCH 25/48] Minor fixes --- b2sdk/api.py | 8 ++++---- b2sdk/replication/setup.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index f043f9b9e..aff61f652 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -538,10 +538,10 @@ def get_key(self, key_id: str) -> Optional[ApplicationKey]: Raises an exception if profile is not permitted to list keys. """ - try: - return self.list_keys(start_application_key_id=key_id)[0] - except IndexError: - return None + return next( + self.list_keys(start_application_key_id=key_id), + None, + ) # other def get_file_info(self, file_id: str) -> FileVersion: diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 2463b60ab..3efe6514d 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -204,13 +204,13 @@ def _get_source_key( do_create_key = True else: current_source_key = api.get_key( - current_replication_configuration.as_replication_source.source_key_id + current_replication_configuration.as_replication_source.source_application_key_id ) if current_source_key is None: do_create_key = True logger.debug( 'will create a new source key because current key %s has been deleted', - current_replication_configuration.source_key_id, + current_replication_configuration.as_replication_source.source_application_key_id, ) else: current_capabilities = current_source_key.get_capabilities() From d126b17d8886058c8569dacfb07065fa5df610dc Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 20 Apr 2022 17:37:34 +0300 Subject: [PATCH 26/48] Lint fix --- b2sdk/replication/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 3efe6514d..56c258149 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -204,13 +204,15 @@ def _get_source_key( do_create_key = True else: current_source_key = api.get_key( - current_replication_configuration.as_replication_source.source_application_key_id + current_replication_configuration.as_replication_source. + source_application_key_id ) if current_source_key is None: do_create_key = True logger.debug( 'will create a new source key because current key %s has been deleted', - current_replication_configuration.as_replication_source.source_application_key_id, + current_replication_configuration.as_replication_source. + source_application_key_id, ) else: current_capabilities = current_source_key.get_capabilities() From 35e549e5e58b3b6b6e5e028ffb67d6f27f111014 Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Wed, 20 Apr 2022 17:37:00 +0300 Subject: [PATCH 27/48] Refactor replication interface --- b2sdk/_v3/__init__.py | 2 +- b2sdk/bucket.py | 7 +-- b2sdk/replication/setting.py | 93 ++++++++++++++++++--------------- test/unit/bucket/test_bucket.py | 2 +- 4 files changed, 58 insertions(+), 46 deletions(-) diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index b02871411..f51afecc5 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -204,7 +204,7 @@ # replication -from b2sdk.replication.setting import ReplicationConfigurationResponse +from b2sdk.replication.setting import ReplicationConfigurationFactory from b2sdk.replication.setting import ReplicationConfiguration from b2sdk.replication.setting import ReplicationSourceConfiguration from b2sdk.replication.setting import ReplicationRule diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 092b4dc6f..e231c17b5 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -31,7 +31,7 @@ ) from .file_version import DownloadVersion, FileVersion from .progress import AbstractProgressListener, DoNothingProgressListener -from .replication.setting import ReplicationConfiguration, ReplicationConfigurationResponse +from .replication.setting import ReplicationConfiguration, ReplicationConfigurationFactory from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.write_intent import WriteIntent from .transfer.inbound.downloaded_file import DownloadedFile @@ -161,6 +161,7 @@ def update( :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting + :param replication: replication rules for the bucket; """ account_id = self.api.account_info.get_account_id() return self.api.BUCKET_FACTORY_CLASS.from_api_bucket_dict( @@ -1066,7 +1067,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) - replication = ReplicationConfigurationResponse.from_bucket_dict(bucket_dict) + replication = ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value return cls.BUCKET_CLASS( api, bucket_id, @@ -1081,5 +1082,5 @@ def from_api_bucket_dict(cls, api, bucket_dict): default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, - replication and replication.value, + replication, ) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 16279b2bc..3857b929e 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -11,7 +11,7 @@ import re from builtins import classmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ClassVar, Dict, List, Optional @@ -69,16 +69,17 @@ class ReplicationSourceConfiguration: Hold information about bucket being a replication source """ - rules: List[ReplicationRule] - source_application_key_id: str + rules: List[ReplicationRule] = field(default_factory=list) + source_application_key_id: Optional[str] = None def __post_init__(self): - if not self.rules: - raise ValueError("rules must not be empty") - - if not self.source_application_key_id: + if self.rules and not self.source_application_key_id: raise ValueError("source_application_key_id must not be empty") + def serialize_to_json_for_request(self) -> Optional[dict]: + if self.rules or self.source_application_key_id: + return self.as_dict() + def as_dict(self) -> dict: return { "replicationRules": [rule.as_dict() for rule in self.rules], @@ -97,19 +98,20 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationSourceConfiguration': @dataclass class ReplicationDestinationConfiguration: - source_to_destination_key_mapping: Dict[str, str] + source_to_destination_key_mapping: Dict[str, str] = field(default_factory=dict) def __post_init__(self): - if not self.source_to_destination_key_mapping: - raise ValueError("source_to_destination_key_mapping must not be empty") - for source, destination in self.source_to_destination_key_mapping.items(): if not source or not destination: raise ValueError( "source_to_destination_key_mapping must not contain \ - empty keys or values: ({}, {})".format(source, destination) + empty keys or values: ({}, {})".format(source, destination) ) + def serialize_to_json_for_request(self) -> Optional[dict]: + if self.source_to_destination_key_mapping: + return self.as_dict() + def as_dict(self) -> dict: return { 'sourceToDestinationKeyMapping': self.source_to_destination_key_mapping, @@ -126,8 +128,12 @@ class ReplicationConfiguration: Hold information about bucket replication configuration """ - as_replication_source: Optional[ReplicationSourceConfiguration] = None - as_replication_destination: Optional[ReplicationDestinationConfiguration] = None + as_replication_source: ReplicationSourceConfiguration = field( + default_factory=ReplicationSourceConfiguration, + ) + as_replication_destination: ReplicationDestinationConfiguration = field( + default_factory=ReplicationDestinationConfiguration, + ) def serialize_to_json_for_request(self) -> dict: return self.as_dict() @@ -168,29 +174,24 @@ def as_dict(self) -> dict: """ - result = {} - - if self.as_replication_source: - result['asReplicationSource'] = self.as_replication_source.as_dict() - - if self.as_replication_destination: - result['asReplicationDestination'] = self.as_replication_destination.as_dict() - - return result + return { + 'asReplicationSource': + self.as_replication_source.serialize_to_json_for_request(), + 'asReplicationDestination': + self.as_replication_destination.serialize_to_json_for_request(), + } @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': - if value_dict is None: - return replication_source_dict = value_dict.get('asReplicationSource') - as_replication_source = replication_source_dict and ReplicationSourceConfiguration.from_dict( + as_replication_source = ReplicationSourceConfiguration.from_dict( replication_source_dict - ) + ) if replication_source_dict else ReplicationSourceConfiguration() replication_destination_dict = value_dict.get('asReplicationDestination') - as_replication_destination = replication_destination_dict and ReplicationDestinationConfiguration.from_dict( + as_replication_destination = ReplicationDestinationConfiguration.from_dict( replication_destination_dict - ) + ) if replication_destination_dict else ReplicationDestinationConfiguration() return cls( as_replication_source=as_replication_source, @@ -199,27 +200,37 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': @dataclass -class ReplicationConfigurationResponse: +class ReplicationConfigurationFactory: is_client_authorized_to_read: bool value: Optional[ReplicationConfiguration] @classmethod - def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurationResponse']: + def from_bucket_dict(cls, bucket_dict: dict) -> Optional['ReplicationConfigurationFactory']: """ - Returns ReplicationConfigurationResponse for the given bucket dict + Returns ReplicationConfigurationFactory for the given bucket dict retrieved from the api, or None if no replication configured. """ - replication_data = bucket_dict.get('replicationConfiguration') - if replication_data is None: - return + replication_dict = bucket_dict.get('replicationConfiguration') + if not replication_dict: + return cls( + is_client_authorized_to_read=True, + value=ReplicationConfiguration(), + ) - return cls.from_dict(bucket_dict['replicationConfiguration']) + return cls.from_dict(replication_dict) @classmethod - def from_dict(cls, value_dict: dict) -> 'ReplicationConfigurationResponse': - if value_dict is None: - return + def from_dict(cls, value_dict: dict) -> 'ReplicationConfigurationFactory': + if not value_dict['isClientAuthorizedToRead']: + return cls( + is_client_authorized_to_read=False, + value=None, + ) + + replication_dict = value_dict['value'] + replication_configuration = ReplicationConfiguration.from_dict(replication_dict) \ + if replication_dict else ReplicationConfiguration() return cls( - is_client_authorized_to_read=value_dict['isClientAuthorizedToRead'], - value=value_dict['value'] and ReplicationConfiguration.from_dict(value_dict['value']), + is_client_authorized_to_read=True, + value=replication_configuration, ) diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 81e3da0b5..83ffe19d0 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -60,7 +60,7 @@ from apiver_deps import BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, RetentionPeriod, \ NO_RETENTION_FILE_SETTING from apiver_deps import ReplicationConfiguration, ReplicationSourceConfiguration, \ - ReplicationRule, ReplicationDestinationConfiguration, ReplicationConfigurationResponse + ReplicationRule, ReplicationDestinationConfiguration pytestmark = [pytest.mark.apiver(from_ver=1)] From 0d14facb9d0dbabad0135ea515ff4d71a27c7d9b Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 Apr 2022 02:05:45 +0200 Subject: [PATCH 28/48] ReplicationSetupHelper almost done on a narrow interface --- b2sdk/bucket.py | 8 +- b2sdk/replication/setup.py | 128 +++++++++++++--------------- test/unit/v_all/test_replication.py | 20 +++-- 3 files changed, 74 insertions(+), 82 deletions(-) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index e231c17b5..9c84e5aa4 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -115,7 +115,7 @@ def get_fresh_state(self) -> 'Bucket': raise BucketIdNotFound(self.id_) return buckets_found[0] - def get_id(self): + def get_id(self) -> str: """ Return bucket ID. @@ -123,7 +123,7 @@ def get_id(self): """ return self.id_ - def set_info(self, new_bucket_info, if_revision_is=None): + def set_info(self, new_bucket_info, if_revision_is=None) -> 'Bucket': """ Update bucket info. @@ -132,7 +132,7 @@ def set_info(self, new_bucket_info, if_revision_is=None): """ return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is) - def set_type(self, bucket_type): + def set_type(self, bucket_type) -> 'Bucket': """ Update bucket type. @@ -150,7 +150,7 @@ def update( default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, - ): + ) -> 'Bucket': """ Update various bucket parameters. diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 56c258149..3873f3a88 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -8,14 +8,16 @@ # ###################################################################### -# b2 replication-setup [--create-keys (both|source|destination|auto)] [--widen-source-key-mode (fail|auto)] [--profile profileName] [--priority int|auto]--destination-profile destinationProfileName sourceBucketPath destinationBucketPath [rrName] +# b2 replication-setup [--profile profileName] --destination-profile destinationProfileName sourceBucketPath destinationBucketName [ruleName] # b2 replication-debug [--profile profileName] [--destination-profile destinationProfileName] bucketPath # b2 replication-status [--profile profileName] [--destination-profile destinationProfileName] [sourceBucketPath] [destinationBucketPath] -# b2 replication-pause [--profile profileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] -# b2 replication-unpause [--profile profileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] -# b2 replication-scan [--profile profileName] [--destination-profile destinationProfileName] [--rrName replicationRuleName] [sourceBucketPath] [destinationBucketPath] -from typing import ClassVar, List, Tuple +# b2 replication-pause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] +# b2 replication-unpause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] +# b2 replication-accept destinationBucketName sourceKeyId [destinationKeyId] +# b2 replication-deny destinationBucketName sourceKeyId + +from typing import ClassVar, List, Optional, Tuple from enum import Enum, auto, unique import logging @@ -27,24 +29,13 @@ logger = logging.getLogger(__name__) -@unique -class ReplicationSetupHelperKeyMode(Enum): - NONE = auto() - AUTO = auto() - SOURCE = auto() - DESTINATION = auto() - BOTH = auto() - - class ReplicationSetupHelper: """ class with various methods that help with repliction management """ - DEFAULT_KEY_MODE: ClassVar[ - ReplicationSetupHelperKeyMode - ] = ReplicationSetupHelperKeyMode.AUTO #: default key mode for setup signatures PRIORITY_OFFSET: ClassVar[int] = 10 #: how far to to put the new rule from the existing rules DEFAULT_PRIORITY: ClassVar[int] = 128 #: what priority to set if there are no preexisting rules MAX_PRIORITY: ClassVar[int] = 255 #: maximum allowed priority of a replication rule DEFAULT_SOURCE_CAPABILITIES = 'readFiles' + DEFAULT_DESTINATION_CAPABILITIES = 'writeFiles' def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): assert source_b2api is not None or destination_b2api is not None @@ -57,28 +48,25 @@ def setup_both( destination_bucket: Bucket, name: str = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule - key_mode: ReplicationSetupHelperKeyMode = DEFAULT_KEY_MODE, - widen_source_key: bool = True, - ): - source_key: ApplicationKey = self.setup_source( + ) -> Tuple[Bucket, Bucket]: + source_bucket = self.setup_source( source_bucket_path, destination_bucket.id_, name, priority, - key_mode, - widen_source_key, ) - assert source_key - # TODO - #self.setup_destination(source_key.id_, destination_bucket, key_mode) + destination_bucket = self.setup_destination( + source_bucket.replication.as_replication_source.source_application_key_id, + destination_bucket, + ) + return source_bucket, destination_bucket def setup_destination( self, - api: B2Api, source_key_id: str, destination_bucket: Bucket, - key_mode: ReplicationSetupHelperKeyMode = DEFAULT_KEY_MODE, - ): + ) -> Bucket: + api: B2Api = destination_bucket.api try: source_configuration = destination_bucket.replication.as_replication_source except (NameError, AttributeError): @@ -88,32 +76,51 @@ def setup_destination( except (NameError, AttributeError): destination_configuration = ReplicationDestinationConfiguration({}) - current_destination_key = destination_configuration.source_to_destination_key_mapping.get( - source_key_id - ) + # TODO: try to find a good key in the mapping, maybe one exists already? OTOH deletions... + #current_destination_key_ids = destination_configuration.source_to_destination_key_mapping.values() - destination_key = self._get_destination_key(current_destination_key) + destination_key = self._get_destination_key( + api, destination_bucket, current_destination_key=None + ) destination_configuration.source_to_destination_key_mapping[source_key_id] = destination_key new_replication_configuration = ReplicationConfiguration( source_configuration, destination_configuration, ) - return new_replication_configuration + return destination_bucket.update( + if_revision_is=destination_bucket.revision, + replication=new_replication_configuration, + ) - def _get_destination_key(self, current_destination_key) -> ApplicationKey: - # name=source_bucket.name[:91] + '-replisrc', - do_create_key = False - if current_destination_key is None: - do_create_key = True - if current_destination_key.prefix: - pass + def _get_destination_key( + self, + api: B2Api, + destination_bucket, + current_destination_key, + ) -> ApplicationKey: + #do_create_key = False + #if current_destination_key is None: + # do_create_key = True + #if current_destination_key.prefix: + # pass # TODO - if not do_create_key: - return current_destination_key + #if not do_create_key: + # return current_destination_key + #else: + return self._create_destination_key( + name=destination_bucket.name[:91] + '-replidst', + api=api, + bucket_id=destination_bucket.id_, + prefix=None, + ) def setup_source( - self, source_bucket_path, destination_bucket_id, name, priority, key_mode, widen_source_key - ) -> ApplicationKey: + self, + source_bucket_path, + destination_bucket_id, + name, + priority, + ) -> Bucket: source_bucket_name, prefix = self._partion_bucket_path(source_bucket_path) source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh @@ -129,8 +136,6 @@ def setup_source( source_key_id = self._get_source_key( source_bucket, prefix, - key_mode, - widen_source_key, source_bucket.replication, current_source_rrs, ) @@ -152,33 +157,21 @@ def setup_source( ), destination_configuration, ) - source_bucket.update( + return source_bucket.update( if_revision_is=source_bucket.revision, replication=new_replication_configuration, ) - return source_key_id @classmethod def _get_source_key( cls, source_bucket, prefix, - key_mode, - widen_source_key, current_replication_configuration: ReplicationConfiguration, current_source_rrs, ) -> str: - assert widen_source_key # TODO api = source_bucket.api - force_source_key_creation: bool = key_mode in ( - ReplicationSetupHelperKeyMode.SOURCE, - ReplicationSetupHelperKeyMode.BOTH, - #ReplicationSetupHelperKeyMode.AUTO, - ) - #assert force_source_key_creation # TODO - #create_source_key_if_needed: bool = key_mode is ReplicationSetupHelperKeyMode.LAZY - do_create_key = False new_prefix = cls._get_narrowest_common_prefix( [rr.path for rr in current_source_rrs] + [prefix] @@ -192,13 +185,6 @@ def _get_source_key( prefix = new_prefix do_create_key = True - if force_source_key_creation: - logger.debug( - 'forced key creation because key_mode is %s', - ReplicationSetupHelperKeyMode(key_mode).name, - ) - do_create_key = True - if not do_create_key: if not current_replication_configuration or not current_replication_configuration.as_replication_source: do_create_key = True @@ -250,7 +236,7 @@ def _create_source_key( name: str, api: B2Api, bucket_id: str, - prefix: str, + prefix: Optional[str] = None, ) -> ApplicationKey: capabilities = cls.DEFAULT_SOURCE_CAPABILITIES return cls._create_key(name, api, bucket_id, prefix, capabilities) @@ -261,7 +247,7 @@ def _create_destination_key( name: str, api: B2Api, bucket_id: str, - prefix: str, + prefix: Optional[str] = None, ) -> ApplicationKey: capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES return cls._create_key(name, api, bucket_id, prefix, capabilities) @@ -272,8 +258,8 @@ def _create_key( name: str, api: B2Api, bucket_id: str, - prefix: str, - capabilities, + prefix: Optional[str] = None, + capabilities=tuple(), ) -> ApplicationKey: return api.create_key( capabilities=capabilities, diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index 56841ace9..f69249cdc 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -8,6 +8,7 @@ # ###################################################################### +import logging import pytest from apiver_deps import B2Api @@ -20,6 +21,8 @@ from b2sdk.replication.setup import ReplicationSetupHelper +logger = logging.getLogger(__name__) + class TestReplication(TestBase): def setUp(self): @@ -39,22 +42,25 @@ def test_get_bucket_by_id_v2(self): self._authorize_account() #with pytest.raises(BucketIdNotFound): # self.api.get_bucket_by_id("this id doesn't even exist") - created_bucket = self.api.create_bucket('bucket1', 'allPrivate') + source_bucket = self.api.create_bucket('bucket1', 'allPrivate') destination_bucket = self.api.create_bucket('bucket2', 'allPrivate') - #read_bucket = self.api.get_bucket_by_id(created_bucket.id_) - #assert created_bucket.id_ == read_bucket.id_ + #read_bucket = self.api.get_bucket_by_id(source_bucket.id_) + #assert source_bucket.id_ == read_bucket.id_ #self.cache.save_bucket(Bucket(api=self.api, name='bucket_name', id_='bucket_id')) #read_bucket = self.api.get_bucket_by_id('bucket_id') #assert read_bucket.name == 'bucket_name' + logger.info('preparations complete, starting the test') rsh = ReplicationSetupHelper( source_b2api=self.api, destination_b2api=self.api, ) - rsh.setup_both( + source_bucket, destination_bucket = rsh.setup_both( source_bucket_path="bucket1", destination_bucket=destination_bucket, name='aa', - #priority=None, - #key_mode, - #widen_source_key, ) + print(source_bucket.replication) + # ReplicationConfiguration(as_replication_source=ReplicationSourceConfiguration(rules=[ReplicationRule(destination_bucket_id='bucket_1', name='aa', file_name_prefix='', is_enabled=True, priority=128)], source_application_key_id=), as_replication_destination=ReplicationDestinationConfiguration(source_to_destination_key_mapping={})) + print('---') + print(destination_bucket.replication) + # ReplicationConfiguration(as_replication_source=ReplicationSourceConfiguration(rules=[], source_application_key_id=None), as_replication_destination=ReplicationDestinationConfiguration(source_to_destination_key_mapping={: })) From 74de0c597b4abadc5dae19e21186e2490566c12c Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 21 Apr 2022 23:08:29 +0200 Subject: [PATCH 29/48] ReplicationSetupHelper operational --- b2sdk/application_key.py | 4 + b2sdk/replication/setup.py | 195 ++++++++++++++++------------ test/unit/v_all/test_replication.py | 94 +++++++++++++- 3 files changed, 201 insertions(+), 92 deletions(-) diff --git a/b2sdk/application_key.py b/b2sdk/application_key.py index 17b4fcffb..27aa4c86b 100644 --- a/b2sdk/application_key.py +++ b/b2sdk/application_key.py @@ -65,6 +65,10 @@ def parse_response_dict(cls, response: dict): for key, value in optional_args.items() if value is not None}, } + def has_capabilities(self, capabilities) -> bool: + """ checks whether the key has ALL of the given capabilities """ + return len(set(capabilities) - set(self.capabilities)) == 0 + def as_dict(self): """Represent the key as a dict, like the one returned by B2 cloud""" mandatory_keys = { diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 3873f3a88..01972c8f6 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -18,24 +18,33 @@ # b2 replication-deny destinationBucketName sourceKeyId from typing import ClassVar, List, Optional, Tuple -from enum import Enum, auto, unique import logging from b2sdk.api import B2Api from b2sdk.application_key import ApplicationKey from b2sdk.bucket import Bucket +from b2sdk.utils import B2TraceMeta from b2sdk.replication.setting import ReplicationConfiguration, ReplicationDestinationConfiguration, ReplicationRule, ReplicationSourceConfiguration logger = logging.getLogger(__name__) -class ReplicationSetupHelper: +class ReplicationSetupHelper(metaclass=B2TraceMeta): """ class with various methods that help with repliction management """ - PRIORITY_OFFSET: ClassVar[int] = 10 #: how far to to put the new rule from the existing rules + PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules DEFAULT_PRIORITY: ClassVar[int] = 128 #: what priority to set if there are no preexisting rules MAX_PRIORITY: ClassVar[int] = 255 #: maximum allowed priority of a replication rule - DEFAULT_SOURCE_CAPABILITIES = 'readFiles' - DEFAULT_DESTINATION_CAPABILITIES = 'writeFiles' + DEFAULT_SOURCE_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( + 'readFiles', + 'readFileLegalHolds', + 'readFileRetentions', + ) + DEFAULT_DESTINATION_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( + 'writeFiles', + 'writeFileLegalHolds', + 'writeFileRetentions', + 'deleteFiles', + ) def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): assert source_b2api is not None or destination_b2api is not None @@ -44,13 +53,15 @@ def __init__(self, source_b2api: B2Api = None, destination_b2api: B2Api = None): def setup_both( self, - source_bucket_path: str, + source_bucket_name: str, destination_bucket: Bucket, name: str = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule + prefix: Optional[str] = None, ) -> Tuple[Bucket, Bucket]: source_bucket = self.setup_source( - source_bucket_path, + source_bucket_name, + prefix, destination_bucket.id_, name, priority, @@ -76,13 +87,14 @@ def setup_destination( except (NameError, AttributeError): destination_configuration = ReplicationDestinationConfiguration({}) - # TODO: try to find a good key in the mapping, maybe one exists already? OTOH deletions... - #current_destination_key_ids = destination_configuration.source_to_destination_key_mapping.values() - - destination_key = self._get_destination_key( - api, destination_bucket, current_destination_key=None + keys_to_purge, destination_key = self._get_destination_key( + api, + destination_bucket, + destination_configuration, ) - destination_configuration.source_to_destination_key_mapping[source_key_id] = destination_key + + destination_configuration.source_to_destination_key_mapping[source_key_id + ] = destination_key.id_ new_replication_configuration = ReplicationConfiguration( source_configuration, destination_configuration, @@ -92,37 +104,56 @@ def setup_destination( replication=new_replication_configuration, ) + @classmethod def _get_destination_key( - self, + cls, api: B2Api, - destination_bucket, - current_destination_key, - ) -> ApplicationKey: - #do_create_key = False - #if current_destination_key is None: - # do_create_key = True - #if current_destination_key.prefix: - # pass - # TODO - #if not do_create_key: - # return current_destination_key - #else: - return self._create_destination_key( - name=destination_bucket.name[:91] + '-replidst', - api=api, - bucket_id=destination_bucket.id_, - prefix=None, + destination_bucket: Bucket, + destination_configuration: ReplicationDestinationConfiguration, + ): + keys_to_purge = [] + current_destination_key_ids = destination_configuration.source_to_destination_key_mapping.values( ) + key = None + for current_destination_key_id in current_destination_key_ids: + # potential inefficiency here as we are fetching keys one by one, however + # the number of keys on an account is limited to a 100 000 000 per account lifecycle + # while the number of keys in the map can be expected to be very low + current_destination_key = api.get_key(current_destination_key_id) + if current_destination_key is None: + logger.debug( + 'zombie key found in replication destination_configuration.source_to_destination_key_mapping: %s', + current_destination_key_id + ) + keys_to_purge.append(current_destination_key_id) + continue + if current_destination_key.has_capabilities( + cls.DEFAULT_DESTINATION_CAPABILITIES + ) and not current_destination_key.name_prefix: + logger.debug('matching destination key found: %s', current_destination_key_id) + key = current_destination_key + # not breaking here since we want to fill the purge list + if not key: + logger.debug("no matching key found, making a new one") + key = cls._create_destination_key( + name=destination_bucket.name[:91] + '-replidst', + api=api, + bucket_id=destination_bucket.id_, + prefix=None, + ) + return keys_to_purge, key def setup_source( self, - source_bucket_path, + source_bucket_name, + prefix, destination_bucket_id, name, priority, ) -> Bucket: - source_bucket_name, prefix = self._partion_bucket_path(source_bucket_path) - source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh + source_bucket: Bucket = self.source_b2api.list_buckets(source_bucket_name)[0] # fresh! + if prefix is None: + prefix = "" try: current_source_rrs = source_bucket.replication.as_replication_source.rules @@ -133,7 +164,7 @@ def setup_source( except (NameError, AttributeError): destination_configuration = None - source_key_id = self._get_source_key( + source_key = self._get_source_key( source_bucket, prefix, source_bucket.replication, @@ -152,7 +183,7 @@ def setup_source( ) new_replication_configuration = ReplicationConfiguration( ReplicationSourceConfiguration( - source_application_key_id=source_key_id, + source_application_key_id=source_key.id_, rules=current_source_rrs + [new_rr], ), destination_configuration, @@ -169,67 +200,59 @@ def _get_source_key( prefix, current_replication_configuration: ReplicationConfiguration, current_source_rrs, - ) -> str: + ) -> ApplicationKey: api = source_bucket.api - do_create_key = False - new_prefix = cls._get_narrowest_common_prefix( - [rr.path for rr in current_source_rrs] + [prefix] + current_source_key = api.get_key( + current_replication_configuration.as_replication_source.source_application_key_id + ) + do_create_key = cls._should_make_new_source_key( + current_replication_configuration, + current_source_key, ) - if new_prefix != prefix: - logger.debug( - 'forced key creation due to widened key prefix from %s to %s', - prefix, - new_prefix, - ) - prefix = new_prefix - do_create_key = True - if not do_create_key: - if not current_replication_configuration or not current_replication_configuration.as_replication_source: - do_create_key = True - else: - current_source_key = api.get_key( - current_replication_configuration.as_replication_source. - source_application_key_id - ) - if current_source_key is None: - do_create_key = True - logger.debug( - 'will create a new source key because current key %s has been deleted', - current_replication_configuration.as_replication_source. - source_application_key_id, - ) - else: - current_capabilities = current_source_key.get_capabilities() - if not prefix.startswith(current_source_key.prefix): - do_create_key = True - logger.debug( - 'will create a new source key because %s installed so far does not encompass current needs with its prefix', - current_source_key.id_, - ) - elif 'readFiles' not in current_capabilities: # TODO: more permissions - do_create_key = True - logger.debug( - 'will create a new source key because %s installed so far does not have enough permissions for replication source', - current_source_key.id_, - ) - else: - return current_source_key - - #if not prefix.startswith(current_key.prefix): - # do_create_key = True - #else: - # new_key = current_key + return current_source_key new_key = cls._create_source_key( name=source_bucket.name[:91] + '-replisrc', api=api, bucket_id=source_bucket.id_, - prefix=prefix, - ) + ) # no prefix! return new_key + @classmethod + def _should_make_new_source_key( + cls, + current_replication_configuration: ReplicationConfiguration, + current_source_key: Optional[ApplicationKey], + ) -> bool: + if current_replication_configuration.as_replication_source.source_application_key_id is None: + logger.debug('will create a new source key because no key is set') + return True + + if current_source_key is None: + logger.debug( + 'will create a new source key because current key "%s" was deleted', + current_replication_configuration.as_replication_source.source_application_key_id, + ) + return True + + if current_source_key.name_prefix: + logger.debug( + 'will create a new source key because current key %s had a prefix: "%s"', + current_source_key.name_prefix, + ) + return True + + if not current_source_key.has_capabilities(cls.DEFAULT_SOURCE_CAPABILITIES): + logger.debug( + 'will create a new source key because %s installed so far does not have enough permissions for replication source: ', + current_source_key.id_, + current_source_key.capabilities, + ) + return True + return False # current key is ok + @classmethod def _create_source_key( cls, diff --git a/test/unit/v_all/test_replication.py b/test/unit/v_all/test_replication.py index f69249cdc..8640f40e4 100644 --- a/test/unit/v_all/test_replication.py +++ b/test/unit/v_all/test_replication.py @@ -17,6 +17,7 @@ from apiver_deps import InMemoryCache from apiver_deps import InMemoryAccountInfo from apiver_deps import RawSimulator +from apiver_deps import ReplicationRule, ReplicationDestinationConfiguration, ReplicationSourceConfiguration from ..test_base import TestBase from b2sdk.replication.setup import ReplicationSetupHelper @@ -38,7 +39,7 @@ def _authorize_account(self): self.api.authorize_account('production', self.application_key_id, self.master_key) @pytest.mark.apiver(from_ver=2) - def test_get_bucket_by_id_v2(self): + def test_setup_both(self): self._authorize_account() #with pytest.raises(BucketIdNotFound): # self.api.get_bucket_by_id("this id doesn't even exist") @@ -55,12 +56,93 @@ def test_get_bucket_by_id_v2(self): destination_b2api=self.api, ) source_bucket, destination_bucket = rsh.setup_both( - source_bucket_path="bucket1", + source_bucket_name="bucket1", destination_bucket=destination_bucket, name='aa', + prefix='ab', ) - print(source_bucket.replication) - # ReplicationConfiguration(as_replication_source=ReplicationSourceConfiguration(rules=[ReplicationRule(destination_bucket_id='bucket_1', name='aa', file_name_prefix='', is_enabled=True, priority=128)], source_application_key_id=), as_replication_destination=ReplicationDestinationConfiguration(source_to_destination_key_mapping={})) - print('---') + + from pprint import pprint + pprint([k.as_dict() for k in self.api.list_keys()]) + + keymap = {k.key_name: k for k in self.api.list_keys()} + + source_application_key = keymap['bucket1-replisrc'] + assert source_application_key + assert set(source_application_key.capabilities) == set( + ('readFiles', 'readFileLegalHolds', 'readFileRetentions') + ) + assert not source_application_key.name_prefix + assert source_application_key.expiration_timestamp_millis is None + + destination_application_key = keymap['bucket2-replidst'] + assert destination_application_key + assert set(destination_application_key.capabilities) == set( + ('writeFiles', 'writeFileLegalHolds', 'writeFileRetentions', 'deleteFiles') + ) + assert not destination_application_key.name_prefix + assert destination_application_key.expiration_timestamp_millis is None + + assert source_bucket.replication.as_replication_source == ReplicationSourceConfiguration( + rules=[ + ReplicationRule( + destination_bucket_id='bucket_1', + name='aa', + file_name_prefix='ab', + is_enabled=True, + priority=128, + ) + ], + source_application_key_id=source_application_key.id_, + ) + assert source_bucket.replication.as_replication_destination == ReplicationDestinationConfiguration( + source_to_destination_key_mapping={}, + ) + print(destination_bucket.replication) - # ReplicationConfiguration(as_replication_source=ReplicationSourceConfiguration(rules=[], source_application_key_id=None), as_replication_destination=ReplicationDestinationConfiguration(source_to_destination_key_mapping={: })) + assert destination_bucket.replication.as_replication_source == ReplicationSourceConfiguration( + rules=[], + source_application_key_id=None, + ) + assert destination_bucket.replication.as_replication_destination == ReplicationDestinationConfiguration( + source_to_destination_key_mapping={ + source_application_key.id_: destination_application_key.id_ + } + ) + + old_source_application_key = source_application_key + + source_bucket, destination_bucket = rsh.setup_both( + source_bucket_name="bucket1", + destination_bucket=destination_bucket, + name='ac', + prefix='ad', + ) + + keymap = {k.key_name: k for k in self.api.list_keys()} + new_source_application_key = keymap['bucket1-replisrc'] + assert source_bucket.replication.as_replication_source == ReplicationSourceConfiguration( + rules=[ + ReplicationRule( + destination_bucket_id='bucket_1', + name='aa', + file_name_prefix='ab', + is_enabled=True, + priority=128, + ), + ReplicationRule( + destination_bucket_id='bucket_1', + name='ac', + file_name_prefix='ad', + is_enabled=True, + priority=133, + ), + ], + source_application_key_id=old_source_application_key.id_, + ) + + assert destination_bucket.replication.as_replication_destination == ReplicationDestinationConfiguration( + source_to_destination_key_mapping={ + new_source_application_key.id_: destination_application_key.id_ + } + ) From ab7bb3096aeef91a831896db546a921869e4768a Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Fri, 22 Apr 2022 01:31:12 +0200 Subject: [PATCH 30/48] Expose ReplicationSetupHelper in apiver v2 --- b2sdk/_v3/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/b2sdk/_v3/__init__.py b/b2sdk/_v3/__init__.py index f51afecc5..f91669f1b 100644 --- a/b2sdk/_v3/__init__.py +++ b/b2sdk/_v3/__init__.py @@ -209,6 +209,7 @@ from b2sdk.replication.setting import ReplicationSourceConfiguration from b2sdk.replication.setting import ReplicationRule from b2sdk.replication.setting import ReplicationDestinationConfiguration +from b2sdk.replication.setup import ReplicationSetupHelper # other From c623098c5ba47803d7249a3c59d26fb586d72ea4 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 23 Apr 2022 02:41:21 +0200 Subject: [PATCH 31/48] Fix bad assertion in raw_api.create_bucket --- b2sdk/raw_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index d129faf7e..0c6d9307e 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -705,8 +705,6 @@ def update_bucket( default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): - assert bucket_info is not None or bucket_type is not None - kwargs = {} if if_revision_is is not None: kwargs['ifRevisionIs'] = if_revision_is @@ -728,6 +726,8 @@ def update_bucket( if replication is not None: kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() + assert kwargs + return self._post_json( api_url, 'b2_update_bucket', From c5b6f5aedf59b88f8623680ffcd54a8ad11ff01d Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 23 Apr 2022 02:42:21 +0200 Subject: [PATCH 32/48] Add support of includeExistingFiles to ReplicationSourceConfiguration --- b2sdk/replication/setting.py | 3 +++ test/integration/test_raw_api.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 3857b929e..e79b587ce 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -27,6 +27,7 @@ class ReplicationRule: file_name_prefix: str = '' is_enabled: bool = True priority: int = 1 + include_existing_files: bool = False REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') @@ -47,6 +48,7 @@ def as_dict(self) -> dict: return { 'destinationBucketId': self.destination_bucket_id, 'fileNamePrefix': self.file_name_prefix, + 'includeExistingFiles': self.include_existing_files, 'isEnabled': self.is_enabled, 'priority': self.priority, 'replicationRuleName': self.name, @@ -57,6 +59,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationRule': return cls( destination_bucket_id=value_dict['destinationBucketId'], file_name_prefix=value_dict['fileNamePrefix'], + include_existing_files=value_dict['includeExistingFiles'], is_enabled=value_dict['isEnabled'], priority=value_dict['priority'], name=value_dict['replicationRuleName'], diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 974af7744..8aaa1d114 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -173,6 +173,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): rules=[ ReplicationRule( destination_bucket_id=bucket_id, + include_existing_files=True, name='test-rule', ), ], @@ -192,6 +193,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): { "destinationBucketId": bucket_id, "fileNamePrefix": "", + "includeExistingFiles": True, "isEnabled": True, "priority": 1, "replicationRuleName": "test-rule" From 3d8a2d8c132b179af02daeb08332dc67dc3eb9d8 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Sat, 23 Apr 2022 02:43:55 +0200 Subject: [PATCH 33/48] Add autonaming of replication rules --- b2sdk/api.py | 2 +- b2sdk/replication/setup.py | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/b2sdk/api.py b/b2sdk/api.py index aff61f652..b44ed8413 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -327,7 +327,7 @@ def get_bucket_by_id(self, bucket_id: str) -> Bucket: # There is no such bucket. raise BucketIdNotFound(bucket_id) - def get_bucket_by_name(self, bucket_name): + def get_bucket_by_name(self, bucket_name: str): """ Return the Bucket matching the given bucket_name. diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index 01972c8f6..be0e1b478 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -18,6 +18,7 @@ # b2 replication-deny destinationBucketName sourceKeyId from typing import ClassVar, List, Optional, Tuple +import itertools import logging from b2sdk.api import B2Api @@ -55,14 +56,14 @@ def setup_both( self, source_bucket_name: str, destination_bucket: Bucket, - name: str = None, #: name for the new replication rule + name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: Optional[str] = None, ) -> Tuple[Bucket, Bucket]: source_bucket = self.setup_source( source_bucket_name, prefix, - destination_bucket.id_, + destination_bucket, name, priority, ) @@ -147,7 +148,7 @@ def setup_source( self, source_bucket_name, prefix, - destination_bucket_id, + destination_bucket: Bucket, name, priority, ) -> Bucket: @@ -174,11 +175,15 @@ def setup_source( priority, current_source_rrs, ) - + name = self._get_name_for_new_rule( + name, + current_source_rrs, + destination_bucket.name, + ) new_rr = ReplicationRule( name=name, priority=priority, - destination_bucket_id=destination_bucket_id, + destination_bucket_id=destination_bucket.id_, file_name_prefix=prefix, ) new_replication_configuration = ReplicationConfiguration( @@ -308,6 +313,28 @@ def _get_priority_for_new_rule(cls, priority, current_source_rrs): return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY) return cls.DEFAULT_PRIORITY + @classmethod + def _get_name_for_new_rule( + cls, name: Optional[str], current_source_rrs, destination_bucket_name + ): + if name is not None: + return name + existing_names = set(rr.name for rr in current_source_rrs) + suffixes = cls._get_rule_name_candidate_suffixes() + while True: + candidate = '%s%s' % (destination_bucket_name, next(suffixes)) + if candidate not in existing_names: + return candidate + + @classmethod + def _get_rule_name_candidate_suffixes(cls): + """ + >>> a = ReplicationSetupHelper._get_rule_name_candidate_suffixes() + >>> [next(a) for i in range(10)] + ['', '2', '3', '4', '5', '6', '7', '8', '9', '10'] + """ + return map(str, itertools.chain([''], itertools.count(2))) + @classmethod def _partion_bucket_path(cls, bucket_path: str) -> Tuple[str, str]: bucket_name, _, path = bucket_path.partition('/') From 2acf553451d57411f0337e642ffbdbdf88f0e372 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 00:25:18 +0200 Subject: [PATCH 34/48] Properly handle empty replications --- b2sdk/raw_simulator.py | 3 +++ b2sdk/replication/setting.py | 13 ++++++------- test/unit/bucket/test_bucket.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 9e5211df6..d7634b5be 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -526,6 +526,9 @@ def __init__( self.is_file_lock_enabled = is_file_lock_enabled self.default_retention = NO_RETENTION_BUCKET_SETTING self.replication = replication + if self.replication is not None: + assert self.replication.asReplicationSource is None or self.replication.asReplicationSource.rules + assert self.replication.asReplicationDestination is None or self.replication.asReplicationDestination.sourceToDestinationKeyMapping def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketEncryption') diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index e79b587ce..b36acce6e 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -176,13 +176,12 @@ def as_dict(self) -> dict: } """ - - return { - 'asReplicationSource': - self.as_replication_source.serialize_to_json_for_request(), - 'asReplicationDestination': - self.as_replication_destination.serialize_to_json_for_request(), - } + result = {} + if self.as_replication_source is not None: + result['asReplicationSource'] = self.as_replication_source.serialize_to_json_for_request() + if self.as_replication_destination is not None: + result['asReplicationDestination'] = self.as_replication_destination.serialize_to_json_for_request() + return result @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 83ffe19d0..3d43fa7c3 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -1033,6 +1033,19 @@ def test_update(self): print(getattr(result, attr_name)) self.assertEqual(attr_value, getattr(result, attr_name), attr_name) + @pytest.mark.apiver(from_ver=2) + def test_empty_replication(self): + self.bucket.update( + replication=ReplicationConfiguration( + as_replication_source=ReplicationSourceConfiguration( + rules=[], + ), + as_replication_destination=ReplicationDestinationConfiguration( + source_to_destination_key_mapping={}, + ), + ), + ) + def test_update_if_revision_is(self): current_revision = self.bucket.revision self.bucket.update( From 4a22368a2c5054f1dd2bfd00cf6aebf30febfb66 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 00:35:11 +0200 Subject: [PATCH 35/48] Handle empty replication json arguments --- b2sdk/replication/setting.py | 22 ++++++++++++---------- test/unit/bucket/test_bucket.py | 4 +--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index b36acce6e..69517e130 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -56,14 +56,14 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationRule': - return cls( - destination_bucket_id=value_dict['destinationBucketId'], - file_name_prefix=value_dict['fileNamePrefix'], - include_existing_files=value_dict['includeExistingFiles'], - is_enabled=value_dict['isEnabled'], - priority=value_dict['priority'], - name=value_dict['replicationRuleName'], - ) + kwargs = {} + kwargs['destination_bucket_id'] = value_dict['destinationBucketId'] + kwargs['name'] = value_dict['replicationRuleName'] + kwargs['file_name_prefix'] = value_dict.get('fileNamePrefix') + kwargs['include_existing_files'] = value_dict.get('includeExistingFiles') + kwargs['is_enabled'] = value_dict.get('isEnabled') + kwargs['priority'] = value_dict.get('priority') + return cls(**kwargs) @dataclass @@ -178,9 +178,11 @@ def as_dict(self) -> dict: """ result = {} if self.as_replication_source is not None: - result['asReplicationSource'] = self.as_replication_source.serialize_to_json_for_request() + result['asReplicationSource' + ] = self.as_replication_source.serialize_to_json_for_request() if self.as_replication_destination is not None: - result['asReplicationDestination'] = self.as_replication_destination.serialize_to_json_for_request() + result['asReplicationDestination' + ] = self.as_replication_destination.serialize_to_json_for_request() return result @classmethod diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 3d43fa7c3..f9c421e02 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -1037,9 +1037,7 @@ def test_update(self): def test_empty_replication(self): self.bucket.update( replication=ReplicationConfiguration( - as_replication_source=ReplicationSourceConfiguration( - rules=[], - ), + as_replication_source=ReplicationSourceConfiguration(rules=[],), as_replication_destination=ReplicationDestinationConfiguration( source_to_destination_key_mapping={}, ), From 03d56ae381d20ef437be3109afbf5f4ac16b3075 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 23:13:51 +0200 Subject: [PATCH 36/48] Add log tracing of interpret_b2_error() --- CHANGELOG.md | 1 + b2sdk/exception.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1113262..c58c48263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ of work on sdk interface: * Add basic replication support to `Bucket` and `FileVersion` * Add `is_master_key()` method to `AbstractAccountInfo` * Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` +* Add log tracing of `interpret_b2_error` ### Fixed * Fix license test on Windows diff --git a/b2sdk/exception.py b/b2sdk/exception.py index 52ce8cf1b..bc5bc206c 100644 --- a/b2sdk/exception.py +++ b/b2sdk/exception.py @@ -10,10 +10,11 @@ from abc import ABCMeta +import logging import re from typing import Any, Dict, Optional -from .utils import camelcase_to_underscore +from .utils import camelcase_to_underscore, trace_call UPLOAD_TOKEN_USED_CONCURRENTLY_ERROR_MESSAGE_RE = re.compile( r'^more than one upload using auth token (?P[^)]+)$' @@ -21,6 +22,8 @@ COPY_SOURCE_TOO_BIG_ERROR_MESSAGE_RE = re.compile(r'^Copy source too big: (?P[\d]+)$') +logger = logging.getLogger(__name__) + class B2Error(Exception, metaclass=ABCMeta): def __init__(self, *args, **kwargs): @@ -514,6 +517,7 @@ class CopyArgumentsMismatch(InvalidUserInput): pass +@trace_call(logger) def interpret_b2_error( status: int, code: Optional[str], From 654d155d34b06bf8d42a65e02b1aa9885d863b6c Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 23:15:21 +0200 Subject: [PATCH 37/48] Fix unclear errors when running integration tests with a non-full key --- CHANGELOG.md | 1 + test/integration/test_raw_api.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c58c48263..30def2864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ of work on sdk interface: ### Fixed * Fix license test on Windows +* Fix unclear errors when running integration tests with a non-full key ## [1.15.0] - 2022-04-12 diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 8aaa1d114..978fec6f4 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -22,7 +22,7 @@ from b2sdk.replication.setting import ReplicationConfiguration, ReplicationSourceConfiguration, ReplicationRule, ReplicationDestinationConfiguration from b2sdk.replication.types import ReplicationStatus from b2sdk.file_lock import BucketRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod -from b2sdk.raw_api import B2RawHTTPApi, REALM_URLS +from b2sdk.raw_api import ALL_CAPABILITIES, B2RawHTTPApi, REALM_URLS from b2sdk.utils import hex_sha1_of_stream @@ -90,6 +90,11 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): # b2_authorize_account print('b2_authorize_account') auth_dict = authorize_raw_api(raw_api) + missing_capabilities = set(ALL_CAPABILITIES) - set(auth_dict['allowed']['capabilities']) + assert not missing_capabilities, 'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: %s' % ( + missing_capabilities, + ) + account_id = auth_dict['accountId'] account_auth_token = auth_dict['authorizationToken'] api_url = auth_dict['apiUrl'] From 1faa9d9145601279f90383388e9f647d78fe728f Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 23:17:36 +0200 Subject: [PATCH 38/48] Fix assertion in integration tests --- test/integration/test_raw_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 978fec6f4..853266a44 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -210,7 +210,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): }, } - # # 3) upload test file and check replication status + # 3) upload test file and check replication status upload_url_dict = raw_api.get_upload_url( api_url, account_auth_token, @@ -228,7 +228,8 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): io.BytesIO(file_contents), ) - assert ReplicationStatus[file_dict['replicationStatus'].upper()] + assert ReplicationStatus[file_dict['replicationStatus'].upper() + ] == ReplicationStatus.PENDING finally: raw_api.delete_key(api_url, account_auth_token, replication_source_key) From 67105b0e7d0d873ad9cff354cf52a0b3c222c8ca Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 23:29:42 +0200 Subject: [PATCH 39/48] Cleanup in replication.setting --- b2sdk/replication/setting.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 69517e130..52851465c 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -35,12 +35,6 @@ def __post_init__(self): if not self.destination_bucket_id: raise ValueError('destination_bucket_id is required') - # TODO - # if not (1 < self.priority < 255): - # raise ValueError() - - # TODO: file_name_prefix validation - if not self.REPLICATION_RULE_REGEX.match(self.name): raise ValueError('replication rule name is invalid') @@ -57,12 +51,19 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationRule': kwargs = {} - kwargs['destination_bucket_id'] = value_dict['destinationBucketId'] - kwargs['name'] = value_dict['replicationRuleName'] - kwargs['file_name_prefix'] = value_dict.get('fileNamePrefix') - kwargs['include_existing_files'] = value_dict.get('includeExistingFiles') - kwargs['is_enabled'] = value_dict.get('isEnabled') - kwargs['priority'] = value_dict.get('priority') + for field, protocolField in ( + ('destination_bucket_id', 'destinationBucketId'), + ('name', 'replicationRuleName'), + ('file_name_prefix', 'fileNamePrefix'), + ('include_existing_files', 'includeExistingFiles'), + ('is_enabled', 'isEnabled'), + ('priority', 'priority'), + ): + value = value_dict.get( + protocolField + ) # refactor to := when dropping Python 3.7, maybe even dict expression + if value is not None: + kwargs[field] = value return cls(**kwargs) From ce788a525fe485753b39208160234e15d34e48b1 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Apr 2022 01:10:29 +0200 Subject: [PATCH 40/48] Pyflakes --- b2sdk/replication/setting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 52851465c..ae84a3d2b 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -51,7 +51,7 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationRule': kwargs = {} - for field, protocolField in ( + for field_, protocolField in ( ('destination_bucket_id', 'destinationBucketId'), ('name', 'replicationRuleName'), ('file_name_prefix', 'fileNamePrefix'), @@ -63,7 +63,7 @@ def from_dict(cls, value_dict: dict) -> 'ReplicationRule': protocolField ) # refactor to := when dropping Python 3.7, maybe even dict expression if value is not None: - kwargs[field] = value + kwargs[field_] = value return cls(**kwargs) From c4f171ff3d36fc31d8cdb9f32b9bfa294b2be637 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Apr 2022 09:42:30 +0200 Subject: [PATCH 41/48] Allow running integration tests on a key with no readBuckets --- test/integration/test_raw_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 853266a44..1fe7c4058 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -90,7 +90,8 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): # b2_authorize_account print('b2_authorize_account') auth_dict = authorize_raw_api(raw_api) - missing_capabilities = set(ALL_CAPABILITIES) - set(auth_dict['allowed']['capabilities']) + missing_capabilities = set(ALL_CAPABILITIES) - set(['readBuckets'] + ) - set(auth_dict['allowed']['capabilities']) assert not missing_capabilities, 'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: %s' % ( missing_capabilities, ) From c71d0d781d3e7acc336ec6e46706a2f653640d1e Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Tue, 26 Apr 2022 00:51:15 +0200 Subject: [PATCH 42/48] Switch integration tests to allPrivate buckets --- test/integration/test_raw_api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 1fe7c4058..aaed2885e 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -139,7 +139,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_name, - 'allPublic', + 'allPrivate', is_file_lock_enabled=True, ) bucket_id = bucket_dict['bucketId'] @@ -172,7 +172,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, replication_source_bucket_name, - 'allPublic', + 'allPrivate', is_file_lock_enabled=True, replication=ReplicationConfiguration( as_replication_source=ReplicationSourceConfiguration( @@ -255,7 +255,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPublic', + 'allPrivate', replication=ReplicationConfiguration( as_replication_destination=ReplicationDestinationConfiguration( source_to_destination_key_mapping={ @@ -291,7 +291,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPublic', + 'allPrivate', replication=ReplicationConfiguration(), ) assert bucket_dict['replicationConfiguration'] == { @@ -327,7 +327,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPublic', + 'allPrivate', default_server_side_encryption=encryption_setting, default_retention=default_retention, ) From f91925b4ef990cb3b62eef214c1b9164ec6d956d Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Tue, 26 Apr 2022 16:37:59 +0300 Subject: [PATCH 43/48] Add `includeExistingFiles` field to docstrings --- b2sdk/bucket.py | 2 ++ b2sdk/replication/setting.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 9c84e5aa4..fea98465c 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -1021,6 +1021,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", + "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" @@ -1028,6 +1029,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", + "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index ae84a3d2b..ef12a5446 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -154,6 +154,7 @@ def as_dict(self) -> dict: { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", + "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" @@ -161,6 +162,7 @@ def as_dict(self) -> dict: { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", + "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" From 5fcf12df28b34f3668c489e3267b985835f9316b Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Apr 2022 19:39:47 +0200 Subject: [PATCH 44/48] Revert "Switch integration tests to allPrivate buckets" This reverts commit c71d0d781d3e7acc336ec6e46706a2f653640d1e. --- test/integration/test_raw_api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index aaed2885e..1fe7c4058 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -139,7 +139,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_name, - 'allPrivate', + 'allPublic', is_file_lock_enabled=True, ) bucket_id = bucket_dict['bucketId'] @@ -172,7 +172,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, replication_source_bucket_name, - 'allPrivate', + 'allPublic', is_file_lock_enabled=True, replication=ReplicationConfiguration( as_replication_source=ReplicationSourceConfiguration( @@ -255,7 +255,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPrivate', + 'allPublic', replication=ReplicationConfiguration( as_replication_destination=ReplicationDestinationConfiguration( source_to_destination_key_mapping={ @@ -291,7 +291,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPrivate', + 'allPublic', replication=ReplicationConfiguration(), ) assert bucket_dict['replicationConfiguration'] == { @@ -327,7 +327,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): account_auth_token, account_id, bucket_id, - 'allPrivate', + 'allPublic', default_server_side_encryption=encryption_setting, default_retention=default_retention, ) From 9e8ec9d5f4c4d041b837765e6adcbf58029e81de Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Apr 2022 19:47:16 +0200 Subject: [PATCH 45/48] Prepare 1.16.0 release --- CHANGELOG.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30def2864..4f5beefa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.16.0-alpha.1] - 2022-04-20 +## [1.16.0] - 2022-04-27 -This is an alpha release. A preview, not for production use. It allows for basic -usage of B2 replication feature (currently in closed beta). The main reason of -releasing it early is to pair it with alpha b2 cli. Expect substantial amount -of work on sdk interface: -* The interface of `Bucket.replication` WILL change +This release contains a preview of replication support. It allows for basic +usage of B2 replication feature (currently in closed beta). + +As the interface of the sdk (and the server api) may change, the replication +support shall be considered PRIVATE interface and should be used with caution. +Please consult the documentation on how to safely use the private api interface. + +Expect substantial amount of work on sdk interface: +* The interface of `ReplicationConfiguration` WILL change * The interface of `FileVersion.replication_status` MIGHT change * The interface of `FileVersionDownload` MIGHT change @@ -21,10 +25,11 @@ of work on sdk interface: * Add `is_master_key()` method to `AbstractAccountInfo` * Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` * Add log tracing of `interpret_b2_error` +* Add `ReplicationSetupHelper` ### Fixed * Fix license test on Windows -* Fix unclear errors when running integration tests with a non-full key +* Fix cryptic errors when running integration tests with a non-full key ## [1.15.0] - 2022-04-12 From 5d69e37ed3c99f967708ddb4b8fb985522a4e1d1 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Wed, 27 Apr 2022 22:30:20 +0200 Subject: [PATCH 46/48] Fix ReplicationSetup not finding existing destination keys --- b2sdk/replication/setup.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index be0e1b478..faa7a9f48 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -79,14 +79,16 @@ def setup_destination( destination_bucket: Bucket, ) -> Bucket: api: B2Api = destination_bucket.api - try: - source_configuration = destination_bucket.replication.as_replication_source - except (NameError, AttributeError): + destination_bucket = api.list_buckets(destination_bucket.name)[0] # fresh! + if destination_bucket.replication is None or destination_bucket.replication.as_replication_source is None: source_configuration = None - try: - destination_configuration = destination_bucket.replication.as_replication_destination - except (NameError, AttributeError): + else: + source_configuration = destination_bucket.replication.as_replication_source + + if destination_bucket.replication is None or destination_bucket.replication.as_replication_destination is None: destination_configuration = ReplicationDestinationConfiguration({}) + else: + destination_configuration = destination_bucket.replication.as_replication_destination keys_to_purge, destination_key = self._get_destination_key( api, @@ -134,6 +136,8 @@ def _get_destination_key( logger.debug('matching destination key found: %s', current_destination_key_id) key = current_destination_key # not breaking here since we want to fill the purge list + else: + logger.info('non-matching destination key found: %s', current_destination_key) if not key: logger.debug("no matching key found, making a new one") key = cls._create_destination_key( From 94c4b88d68cbd970a8716ffa984220c4754a42de Mon Sep 17 00:00:00 2001 From: Aleksandr Goncharov Date: Thu, 28 Apr 2022 13:25:14 +0300 Subject: [PATCH 47/48] Add ReplicationRule.MIN_PRIORITY|MAX_PRIORITY --- b2sdk/replication/setting.py | 10 +++++++++- b2sdk/replication/setup.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index ef12a5446..8d85f328a 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -26,10 +26,12 @@ class ReplicationRule: name: str file_name_prefix: str = '' is_enabled: bool = True - priority: int = 1 + priority: int = 128 include_existing_files: bool = False REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') + MIN_PRIORITY: ClassVar[int] = 1 + MAX_PRIORITY: ClassVar[int] = 2147483647 def __post_init__(self): if not self.destination_bucket_id: @@ -38,6 +40,12 @@ def __post_init__(self): if not self.REPLICATION_RULE_REGEX.match(self.name): raise ValueError('replication rule name is invalid') + if not (self.MIN_PRIORITY <= self.priority <= self.MAX_PRIORITY): + raise ValueError('priority should be within [%d, %d] interval' % ( + self.MIN_PRIORITY, + self.MAX_PRIORITY, + )) + def as_dict(self) -> dict: return { 'destinationBucketId': self.destination_bucket_id, diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index faa7a9f48..b29e3c7de 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -33,8 +33,8 @@ class ReplicationSetupHelper(metaclass=B2TraceMeta): """ class with various methods that help with repliction management """ PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules - DEFAULT_PRIORITY: ClassVar[int] = 128 #: what priority to set if there are no preexisting rules - MAX_PRIORITY: ClassVar[int] = 255 #: maximum allowed priority of a replication rule + DEFAULT_PRIORITY: ClassVar[int] = ReplicationRule.priority #: what priority to set if there are no preexisting rules + MAX_PRIORITY: ClassVar[int] = ReplicationRule.MAX_PRIORITY #: maximum allowed priority of a replication rule DEFAULT_SOURCE_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( 'readFiles', 'readFileLegalHolds', From d195fcd4150dd26322ceb59cfb58cf0375f0209d Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 28 Apr 2022 13:54:58 +0200 Subject: [PATCH 48/48] Refactor default replication rule priority --- b2sdk/replication/setting.py | 14 +++++++++----- b2sdk/replication/setup.py | 7 +++++-- test/integration/test_raw_api.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/b2sdk/replication/setting.py b/b2sdk/replication/setting.py index 8d85f328a..3ea52cba4 100644 --- a/b2sdk/replication/setting.py +++ b/b2sdk/replication/setting.py @@ -22,11 +22,13 @@ class ReplicationRule: prefix and rule name. """ + DEFAULT_PRIORITY: ClassVar[int] = 128 + destination_bucket_id: str name: str file_name_prefix: str = '' is_enabled: bool = True - priority: int = 128 + priority: int = DEFAULT_PRIORITY include_existing_files: bool = False REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') @@ -41,10 +43,12 @@ def __post_init__(self): raise ValueError('replication rule name is invalid') if not (self.MIN_PRIORITY <= self.priority <= self.MAX_PRIORITY): - raise ValueError('priority should be within [%d, %d] interval' % ( - self.MIN_PRIORITY, - self.MAX_PRIORITY, - )) + raise ValueError( + 'priority should be within [%d, %d] interval' % ( + self.MIN_PRIORITY, + self.MAX_PRIORITY, + ) + ) def as_dict(self) -> dict: return { diff --git a/b2sdk/replication/setup.py b/b2sdk/replication/setup.py index b29e3c7de..e374e38c7 100644 --- a/b2sdk/replication/setup.py +++ b/b2sdk/replication/setup.py @@ -33,8 +33,11 @@ class ReplicationSetupHelper(metaclass=B2TraceMeta): """ class with various methods that help with repliction management """ PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules - DEFAULT_PRIORITY: ClassVar[int] = ReplicationRule.priority #: what priority to set if there are no preexisting rules - MAX_PRIORITY: ClassVar[int] = ReplicationRule.MAX_PRIORITY #: maximum allowed priority of a replication rule + DEFAULT_PRIORITY: ClassVar[ + int + ] = ReplicationRule.DEFAULT_PRIORITY #: what priority to set if there are no preexisting rules + MAX_PRIORITY: ClassVar[ + int] = ReplicationRule.MAX_PRIORITY #: maximum allowed priority of a replication rule DEFAULT_SOURCE_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( 'readFiles', 'readFileLegalHolds', diff --git a/test/integration/test_raw_api.py b/test/integration/test_raw_api.py index 1fe7c4058..798a62fbf 100644 --- a/test/integration/test_raw_api.py +++ b/test/integration/test_raw_api.py @@ -201,7 +201,7 @@ def raw_api_test_helper(raw_api, should_cleanup_old_buckets): "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": True, - "priority": 1, + "priority": 128, "replicationRuleName": "test-rule" }, ],