-
Notifications
You must be signed in to change notification settings - Fork 0
/
cpy_agmidas.py
executable file
·106 lines (87 loc) · 3.25 KB
/
cpy_agmidas.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#!/usr/bin/env python3
import paramiko
# https://github.com/jbardin/scp.py
from scp import SCPClient
import argparse
import sys
from pathlib import Path
from multiprocessing import Pool
def createSSHClient(server, port, user, password):
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(server, port, user, password)
return client
def progress(filename, size, sent):
sys.stdout.write("%s progress: %.2f%% \r" % (str(filename, 'utf-8'), float(sent)/float(size)*100) )
def progress4(filename, size, sent, peername):
sys.stdout.write("(%s:%s) %s progress: %.2f%% \r" % (peername[0], peername[1], str(filename, 'utf-8'), float(sent)/float(size)*100) )
def getsubruns(runlist):
filelist=[]
for line in runlist:
run=int(line.strip())
sub=0
subrun='/alpha/agdaq/data/run%05dsub%03d.mid.lz4'%(run,sub)
subfile=Path(subrun)
while subfile.is_file():
filelist.append(subrun)
sub+=1
subrun='/alpha/agdaq/data/run%05dsub%03d.mid.lz4'%(run,sub)
subfile=Path(subrun)
return filelist
def getnewsubruns(runlist):
filelist=[]
localpath='/daq/alpha_data0/acapra/alphag/midasdata/'
remotepath='/eos/experiment/ALPHAg/midasdata_old/'
for line in runlist:
l=line.split()
subrun=str(l[0].strip())
subsize=int(l[1].strip())
subfile=Path(subrun)
localfile=Path(localpath+subfile.parts[-1])
if localfile.is_file():
#print(f'yes {subrun}')
localsize=int(localfile.stat().st_size)
if localsize==subsize:
continue
#print(f'no {subrun}')
filelist.append(remotepath+subrun)
return filelist
def worker(runfile):
server='lxplus.cern.ch'
port=22
user='acapra'
password='password'
ssh = createSSHClient(server, port, user, password)
scp = SCPClient(ssh.get_transport(), progress4=progress4)
#scp = SCPClient(ssh.get_transport(), socket_timeout=None, progress=progress)
dst='/daq/alpha_data0/acapra/alphag/midasdata'
try:
scp.put(runfile.strip(),remote_path=dst)
except FileNotFoundError:
print(f'{runfile} not found')
if __name__=='__main__':
parser = argparse.ArgumentParser(description='copy ALPHA-g MIDASfiles to my workspace')
parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin, help='List of runs')
args = parser.parse_args()
#server='lxplus.cern.ch'
server='alpha00.triumf.ca'
port=22
user='acapra'
password='password'
ssh = createSSHClient(server, port, user, password)
#scp = SCPClient(ssh.get_transport(), progress4=progress4)
scp = SCPClient(ssh.get_transport(), socket_timeout=None, progress=progress)
src=getsubruns(args.infile)
#src=getnewsubruns(args.infile)
#for f in src:
# print(f)
#dst='/afs/cern.ch/user/a/acapra/workspace/agdata'
dst='/daq/alpha_data0/acapra/alphag/midasdata'
print('Ready for transfer')
# recursive for directories
# scp.put(src, recursive=True, remote_path=dst)
#
scp.put(src, remote_path=dst)
#pool = Pool(processes=5)
#pool.map(worker, src)