forked from cernbox/smashbox
/
test_basicSync.py
306 lines (214 loc) · 11.7 KB
/
test_basicSync.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
__doc__ = """ Test basic sync and conflbfnicts: files are modified and deleted by one or both workers (winner and loser); optionally remove local state db on one of the clients (loser).
There are four clients (workers):
- creator - populates the directory initially and also performs a final check
- winner - is syncing its local changes first
- loser - is syncing its local changes second (and optionally it looses the local sync database before doing the sync)
- checker - only performs a final check (without having interacted with the system before)
Note: in 1.6 client conflict files are excluded by default - so they should be never propagated to the server
Note: in 1.5 the exclusion list should be provided separately to the client (FIXME)
Note on effects of removing local state db (1.6):
- any files modified remotely or locally get a conflict if both remote and local replicas exist (FIXME: this could be possibly more refined in the future based on timestamps or content checksums)
- any files not present locally but present remotely will be downloaded (so a deletion won't be propagated)
"""
from smashbox.utilities import *
import glob
import time
filesizeKB = int(config.get('basicSync_filesizeKB',10000))
# True => remove local sync db on the loser
# False => keep the loser
rmLocalStateDB = bool(config.get('basicSync_rmLocalStateDB',False))
# True => use new webdav endpoint (dav/files)
# False => use old webdav endpoint (webdav)
use_new_dav_endpoint = bool(config.get('use_new_dav_endpoint',True))
testsets = [
{ 'basicSync_filesizeKB': 1000,
'basicSync_rmLocalStateDB':False,
'use_new_dav_endpoint':True
},
{ 'basicSync_filesizeKB': 1000,
'basicSync_rmLocalStateDB':False,
'use_new_dav_endpoint':False
},
{ 'basicSync_filesizeKB': 50000,
'basicSync_rmLocalStateDB':False,
'use_new_dav_endpoint':True
},
{ 'basicSync_filesizeKB': 50000,
'basicSync_rmLocalStateDB':False,
'use_new_dav_endpoint':False
},
{ 'basicSync_filesizeKB': 1000,
'basicSync_rmLocalStateDB':True,
'use_new_dav_endpoint':True
},
{ 'basicSync_filesizeKB': 1000,
'basicSync_rmLocalStateDB':True,
'use_new_dav_endpoint':False
},
{ 'basicSync_filesizeKB': 50000,
'basicSync_rmLocalStateDB':True,
'use_new_dav_endpoint':True
},
{ 'basicSync_filesizeKB': 50000,
'basicSync_rmLocalStateDB':True,
'use_new_dav_endpoint':False
}
]
def expect_content(fn,md5):
actual_md5 = md5sum(fn)
error_check(actual_md5 == md5, "inconsistent md5 of %s: expected %s, got %s"%(fn,md5,actual_md5))
def expect_no_deleted_files(d):
expect_deleted_files(d,[])
def expect_deleted_files(d,expected_deleted_files):
actual_deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))
logger.debug('deleted files in %s: %s',d,actual_deleted_files)
error_check(len(expected_deleted_files) == len(actual_deleted_files), "expected %d got %d deleted files"%(len(expected_deleted_files),len(actual_deleted_files)))
for fn in expected_deleted_files:
error_check(any([fn in dfn for dfn in actual_deleted_files]), "expected deleted file for %s not found"%fn)
def expect_conflict_files(d,expected_conflict_files):
actual_conflict_files = get_conflict_files(d)
logger.debug('conflict files in %s: %s',d,actual_conflict_files)
error_check(len(expected_conflict_files) == len(actual_conflict_files), "expected %d got %d conflict files"%(len(expected_conflict_files),len(actual_conflict_files)))
exp_basefns = [os.path.splitext(fn)[0] for fn in expected_conflict_files]
logger.debug(exp_basefns)
logger.debug(actual_conflict_files)
for bfn in exp_basefns:
error_check(any([bfn in fn for fn in actual_conflict_files]), "expected conflict file for %s not found"%bfn)
def expect_no_conflict_files(d):
expect_conflict_files(d,[])
def final_check(d,shared):
""" This is the final check applicable to all workers - this reflects the status of the remote repository so everyone should be in sync.
The only potential differences are with locally generated conflict files.
"""
list_files(d)
expect_content(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'), shared['md5_creator'])
expect_content(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'), shared['md5_loser'])
if not rmLocalStateDB:
expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_loser'])
else:
expect_content(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'), shared['md5_creator']) # in this case, a conflict is created on the loser and file on the server stays the same
expect_content(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'), shared['md5_winner'])
expect_content(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'), shared['md5_winner'])
expect_content(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'), shared['md5_winner']) # a conflict on the loser, server not changed
expect_content(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'), shared['md5_winner']) # a conflict on the loser, server not changed
if not rmLocalStateDB:
expect_no_deleted_files(d) # normally any deleted files should not come back
else:
expect_deleted_files(d, ['TEST_FILE_DELETED_LOSER.dat', 'TEST_FILE_DELETED_WINNER.dat']) # but not TEST_FILE_DELETED_BOTH.dat !
expect_content(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'), shared['md5_creator']) # this file should be downloaded by the loser because it has no other choice (no previous state to compare with)
expect_content(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'), shared['md5_creator']) # this file should be re-uploaded by the loser because it has no other choice (no previous state to compare with)
def finish_if_not_capable():
# Finish the test if some of the prerequisites for this test are not satisfied
if compare_oc_version('10.0', '<') and use_new_dav_endpoint == True:
#Dont test for <= 9.1 with new endpoint, since it is not supported
logger.warn("Skipping test since webdav endpoint is not capable for this server version")
return True
return False
@add_worker
def creator(step):
if finish_if_not_capable():
return
reset_owncloud_account()
reset_rundir()
step(1,'create initial content and sync')
d = make_workdir()
# files *_NONE are not modified by anyone after initial sync
# files *_LOSER are modified by the loser but not by the winner
# files *_WINNER are modified by the winner but not by the loser
# files *_BOTH are modified both by the winner and by the loser (always conflict on the loser)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
shared = reflection.getSharedObject()
shared['md5_creator'] = md5sum(os.path.join(d,'TEST_FILE_MODIFIED_NONE.dat'))
logger.info('md5_creator: %s',shared['md5_creator'])
list_files(d)
run_ocsync(d, use_new_dav_endpoint=use_new_dav_endpoint)
list_files(d)
time.sleep(1)
step(7,'download the repository')
run_ocsync(d,n=3, use_new_dav_endpoint=use_new_dav_endpoint)
time.sleep(1)
step(8,'final check')
final_check(d,shared)
expect_no_conflict_files(d)
@add_worker
def winner(step):
if finish_if_not_capable():
return
step(2,'initial sync')
d = make_workdir()
run_ocsync(d, use_new_dav_endpoint=use_new_dav_endpoint)
step(3,'modify locally and sync to server')
list_files(d)
remove_file(os.path.join(d,'TEST_FILE_DELETED_WINNER.dat'))
remove_file(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'))
createfile(os.path.join(d,'TEST_FILE_MODIFIED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'1',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'),'1',count=1000,bs=filesizeKB)
shared = reflection.getSharedObject()
shared['md5_winner'] = md5sum(os.path.join(d,'TEST_FILE_ADDED_WINNER.dat'))
logger.info('md5_winner: %s',shared['md5_winner'])
run_ocsync(d, use_new_dav_endpoint=use_new_dav_endpoint)
time.sleep(1)
step(5,'final sync')
run_ocsync(d,n=3, use_new_dav_endpoint=use_new_dav_endpoint)
time.sleep(1)
step(8,'final check')
final_check(d,shared)
expect_no_conflict_files(d)
# this is the loser which lost it's local state db after initial sync
@add_worker
def loser(step):
if finish_if_not_capable():
return
step(2,'initial sync')
d = make_workdir()
run_ocsync(d, use_new_dav_endpoint=use_new_dav_endpoint)
step(4,'modify locally and sync to the server')
list_files(d)
# now do the local changes
remove_file(os.path.join(d,'TEST_FILE_DELETED_LOSER.dat'))
remove_file(os.path.join(d,'TEST_FILE_DELETED_BOTH.dat'))
createfile(os.path.join(d,'TEST_FILE_MODIFIED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_MODIFIED_BOTH.dat'),'2',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'),'2',count=1000,bs=filesizeKB)
createfile(os.path.join(d,'TEST_FILE_ADDED_BOTH.dat'),'2',count=1000,bs=filesizeKB)
shared = reflection.getSharedObject()
shared['md5_loser'] = md5sum(os.path.join(d,'TEST_FILE_ADDED_LOSER.dat'))
logger.info('md5_loser: %s',shared['md5_loser'])
#os.system('curl -v -s -k -XPROPFIND --data @/b/eos/CURL-TEST/p2.dat %s| xmllint --format -'%oc_webdav_url(remote_folder='TEST_FILE_MODIFIED_BOTH.dat'))
#os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')
# remove the sync db
if rmLocalStateDB:
remove_db_in_folder(d)
run_ocsync(d,n=3, use_new_dav_endpoint=use_new_dav_endpoint) # conflict file will be synced to the server but it requires more than one sync run
time.sleep(1)
step(6,'final sync')
run_ocsync(d, use_new_dav_endpoint=use_new_dav_endpoint)
time.sleep(1)
step(8,'final check')
#os.system('sqlite3 -line /tmp/smashdir/test_basicSync/loser/.csync_journal.db \'select * from metadata where path like "%TEST_FILE_MODIFIED_BOTH%"\'')
final_check(d,shared)
if not rmLocalStateDB:
expect_conflict_files(d, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat' ])
else:
expect_conflict_files(d, ['TEST_FILE_ADDED_BOTH.dat', 'TEST_FILE_MODIFIED_BOTH.dat',
'TEST_FILE_MODIFIED_LOSER.dat', 'TEST_FILE_MODIFIED_WINNER.dat']) # because the local and remote state is different and it is assumed that this is a conflict (FIXME: in the future timestamp-based last-restort check could improve this situation)
@add_worker
def checker(step):
if finish_if_not_capable():
return
shared = reflection.getSharedObject()
step(7,'download the repository for final verification')
d = make_workdir()
run_ocsync(d,n=3, use_new_dav_endpoint=use_new_dav_endpoint)
time.sleep(1)
step(8,'final check')
final_check(d,shared)
expect_no_conflict_files(d)