-
Notifications
You must be signed in to change notification settings - Fork 303
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3710 from cserf/patch-3692-Add_dirac_add_files_me…
…thod_for_the_Rucio_Dirac_integration Add dirac_add_files method for the Rucio/Dirac integration + Belleii : Adding permission file + Define extract_scope based on policy : Closes #3679, #3690, #3692
- Loading branch information
Showing
16 changed files
with
1,694 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| #!/usr/bin/env python | ||
| # Copyright 2020 CERN | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| # Authors: | ||
| # - Cedric Serfon <cedric.serfon@cern.ch>, 2020 | ||
| # | ||
| # PY3K COMPATIBLE | ||
|
|
||
| from __future__ import print_function | ||
|
|
||
| from rucio.api.permission import has_permission | ||
|
|
||
| from rucio.core.rse import get_rse_id | ||
| from rucio.core.scope import list_scopes | ||
| from rucio.core import dirac | ||
| from rucio.common.exception import AccessDenied | ||
| from rucio.common.types import InternalAccount | ||
| from rucio.common.utils import extract_scope | ||
|
|
||
|
|
||
| def add_files(lfns, issuer, ignore_availability): | ||
| """ | ||
| Bulk add files : | ||
| - Create the file and replica. | ||
| - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. | ||
| - Create all the ascendants of the dataset if they do not exist | ||
| :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} | ||
| :param issuer: The issuer account. | ||
| :param ignore_availability: A boolean to ignore blacklisted sites. | ||
| """ | ||
| scopes = list_scopes() | ||
| dids = [] | ||
| rses = {} | ||
| for lfn in lfns: | ||
| scope, name = extract_scope(lfn['lfn'], scopes) | ||
| dids.append({'scope': scope, 'name': name}) | ||
| rse = lfn['rse'] | ||
| if rse not in rses: | ||
| rse_id = get_rse_id(rse=rse) | ||
| rses[rse] = rse_id | ||
| lfn['rse_id'] = rses[rse] | ||
|
|
||
| # Check if the issuer can add dids and use skip_availabitlity | ||
| for rse in rses: | ||
| rse_id = rses[rse] | ||
| kwargs = {'rse': rse, 'rse_id': rse_id} | ||
| if not has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs): | ||
| raise AccessDenied('Account %s can not add file replicas on %s' % (issuer, rse)) | ||
| if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs): | ||
| ignore_availability = False | ||
|
|
||
| # Check if the issuer can add the files | ||
| kwargs = {'issuer': issuer, 'dids': dids} | ||
| if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs): | ||
| raise AccessDenied('Account %s can not bulk add data identifier' % (issuer)) | ||
|
|
||
| issuer = InternalAccount(issuer) | ||
| dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, session=None) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| #!/usr/bin/env python | ||
| # Copyright 2020 CERN | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| # Authors: | ||
| # - Cedric Serfon <cedric.serfon@cern.ch>, 2020 | ||
| # | ||
| # PY3K COMPATIBLE | ||
|
|
||
| from __future__ import print_function | ||
|
|
||
| from json import dumps | ||
| from requests.status_codes import codes | ||
|
|
||
| from rucio.client.baseclient import BaseClient | ||
| from rucio.client.baseclient import choice | ||
| from rucio.common.utils import build_url | ||
|
|
||
|
|
||
| class DiracClient(BaseClient): | ||
|
|
||
| """DataIdentifier client class for working with data identifiers""" | ||
|
|
||
| DIRAC_BASEURL = 'dirac' | ||
|
|
||
| def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, | ||
| auth_type=None, creds=None, timeout=600, user_agent='rucio-clients'): | ||
| super(DiracClient, self).__init__(rucio_host, auth_host, account, ca_cert, | ||
| auth_type, creds, timeout, user_agent) | ||
|
|
||
| def add_files(self, lfns, ignore_availability=False): | ||
| """ | ||
| Bulk add files : | ||
| - Create the file and replica. | ||
| - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. | ||
| - Create all the ascendants of the dataset if they do not exist | ||
| :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} | ||
| :param ignore_availability: A boolean to ignore blacklisted sites. | ||
| """ | ||
| path = '/'.join([self.DIRAC_BASEURL, 'addfiles']) | ||
| url = build_url(choice(self.list_hosts), path=path) | ||
| r = self._send_request(url, type='POST', data=dumps({'lfns': lfns, 'ignore_availability': ignore_availability})) | ||
| if r.status_code == codes.created: | ||
| return True | ||
| else: | ||
| exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content) | ||
| raise exc_cls(exc_msg) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.