/
rsync.py
128 lines (113 loc) · 5.46 KB
/
rsync.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import sh
from dlrn.config import getConfigOptions
logger = logging.getLogger("dlrn-rsync")
def sync_repo(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We are only rsyncing the current repo dir to rsyncdest
rsyncpaths = []
# We are inserting a dot in the path after repos, this is used by
# rsync -R (see man rsync)
commitdir_abs = os.path.join(datadir, "repos", ".",
commit.getshardedcommitdir())
rsyncpaths.append(commitdir_abs)
# We also need report.html, status_report.html, queue.html,
# styles.css and the consistent and current symlinks
for filename in ['report.html', 'status_report.html', 'styles.css',
'queue.html', 'status_report.csv']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
try:
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
rsyncpaths, rsyncdest)
except Exception as e:
logger.warn('Failed to rsync content to %s ,'
'got error %s' % (rsyncdest, e))
# Raise exception, so it can be treated as an error
raise e
def sync_symlinks(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
reponame = config_options.reponame
if rsyncdest != '':
# We want to sync the symlinks in a second pass, once all content
# has been copied, to avoid a race condition it they are copied first
rsyncpaths = []
for filename in ['consistent', 'current']:
if config_options.use_components:
filepath = os.path.join(datadir, "repos", ".", "component",
commit.component, filename)
else:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
# If using components, current and consistent on the top-level dir
# are not symlinks, but full directories
if config_options.use_components:
exclude_list = []
extra_include_list = []
for filename in ['consistent', 'current']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
exclude_list.extend(['--exclude', '%s/%s.repo' %
(filename, reponame)])
exclude_list.extend(['--exclude', '%s/%s.repo.md5' %
(filename, reponame)])
exclude_list.extend(['--exclude', '%s/versions.csv' %
filename])
extra_include_list.append('%s/%s.repo' % (filepath, reponame))
extra_include_list.append('%s/%s.repo.md5' %
(filepath, reponame))
extra_include_list.append('%s/versions.csv' % filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
if config_options.use_components:
try:
# First, rsync everything except the top-level symlinks
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
exclude_list,
rsyncpaths, rsyncdest)
except Exception as e:
# We are not raising exceptions for symlink rsyncs, these will
# be fixed after another build
logger.warn('Failed to rsync symlinks to %s ,'
'got error %s' % (rsyncdest, e))
try:
# Then, the top-level symlinks
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
extra_include_list, rsyncdest)
except Exception as e:
# We are not raising exceptions for symlink rsyncs, these will
# be fixed after another build
logger.warn('Failed to rsync symlinks to %s ,'
'got error %s' % (rsyncdest, e))
else:
try:
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
rsyncpaths, rsyncdest)
except Exception as e:
# We are not raising exceptions for symlink rsyncs, these will
# be fixed after another build
logger.warn('Failed to rsync symlinks to %s ,'
'got error %s' % (rsyncdest, e))