-
Notifications
You must be signed in to change notification settings - Fork 106
/
MakeRucioMockFile.py
85 lines (70 loc) · 2.91 KB
/
MakeRucioMockFile.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Script to produce a json file to mock Rucio data
"""
from __future__ import (division, print_function)
import json
import os
import sys
from future.utils import viewitems
from WMCore.Services.Rucio.Rucio import Rucio
from WMCore.WMBase import getTestBase
### Here goes a list of data that we want to fetch from Rucio and
### persist in our json file to mock those calls/data
CONTAINERS = [u"/MinimumBias/ComissioningHI-v1/RAW",
u"/Cosmics/ComissioningHI-PromptReco-v1/RECO"]
BLOCKS = []
### The output file which will contain all the Rucio mock data
OUTPUT_FILE = "RucioMockData.json"
### some default constants (which will probably not change in a long time)
SCOPE = "cms"
RUCIO_URL = 'http://cms-rucio.cern.ch'
RUCIO_AUTH_URL = 'https://cms-rucio-auth.cern.ch'
RUCIO_ACCT = "wmcore_transferor"
def main():
finalResult = {}
rucio = Rucio(RUCIO_ACCT,
hostUrl=RUCIO_URL,
authUrl=RUCIO_AUTH_URL,
configDict={"auth_type": "x509"})
realRucio = rucio.cli
outFilename = os.path.join(getTestBase(), '..', 'data', 'Mock', OUTPUT_FILE)
calls = []
for container in CONTAINERS:
print("Building call list for container: {}".format(container))
calls.append(['getBlocksInContainer', {'container': container}])
calls.append(['isContainer', {'didName': container}])
calls.append(['getDID', {'didName': container, 'dynamic': False}])
calls.append(['didExist', {'didName': container}])
blocks = realRucio.list_content(scope=SCOPE, name=container)
for block in blocks:
if block['type'].upper() != 'DATASET':
continue
calls.append(['isContainer', {'didName': block['name']}])
calls.append(['getDID', {'didName': block['name'], 'dynamic': False}])
calls.append(['didExist', {'didName': block['name']}])
for block in BLOCKS:
calls.append(['isContainer', {'didName': block}])
calls.append(['getDID', {'didName': block, 'dynamic': False}])
calls.append(['didExist', {'didName': block}])
nCalls = len(calls)
print("Need to issue {} calls to Rucio".format(nCalls))
callsDone = 0
for call in calls:
func = getattr(rucio, call[0])
if len(call) > 1:
signature = '%s:%s' % (call[0], sorted(viewitems(call[1])))
result = func(**call[1])
else:
result = func()
signature = call[0]
callsDone += 1
if callsDone % 10 == 0:
print("Performed {} out of {} calls".format(callsDone, nCalls))
finalResult.update({signature: result})
print("Done!\n\nWriting out {} file".format(outFilename))
with open(outFilename, 'w') as fileObj:
json.dump(finalResult, fileObj, indent=1, separators=(',', ': '), sort_keys=True)
if __name__ == '__main__':
sys.exit(main())