Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 496 lines (424 sloc) 13.506 kb
65dc42b rofl0r initial commit
authored
1 /*
2 Copyright (C) 2012 rofl0r
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License along
15 with this program; if not, write to the Free Software Foundation, Inc.,
16 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18 */
8ccc041 rofl0r print crc in network order
authored
19 #undef _GNU_SOURCE
20 #define _GNU_SOURCE
21 #include "../lib/include/timelib.h"
65dc42b rofl0r initial commit
authored
22 #include "../lib/include/filelist.h"
23 #include "../lib/include/filelib.h"
24 #include "../lib/include/strlib.h"
25 #include "../lib/include/logger.h"
78b1feb rofl0r multiple fixes
authored
26 #include "../lib/include/optparser.h"
65dc42b rofl0r initial commit
authored
27 #include "../lib/include/crc32c.h"
28 #include <fcntl.h>
29 #include <stdio.h>
30 #include <sys/stat.h>
31 #include <errno.h>
32 #include <utime.h>
8ccc041 rofl0r print crc in network order
authored
33 #include <arpa/inet.h>
65dc42b rofl0r initial commit
authored
34
35 typedef struct {
36 uint64_t symlink;
37 uint64_t copies;
38 uint64_t skipped;
39 uint64_t copied;
40 } totals;
41
42 typedef struct {
43 stringptr srcdir_b;
44 stringptr* srcdir;
45 stringptr dstdir_b;
46 stringptr* dstdir;
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
47 stringptr diffdir_b;
48 stringptr* diffdir;
65dc42b rofl0r initial commit
authored
49
50 totals total;
51
52 int doChecksum:1;
78b1feb rofl0r multiple fixes
authored
53 int checkExists:1;
65dc42b rofl0r initial commit
authored
54 int checkChecksum:1;
55 int checkFileSize:1;
56 int checkDate:1;
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
57 int checkDateOlder:1;
78b1feb rofl0r multiple fixes
authored
58 int simulate:1;
65dc42b rofl0r initial commit
authored
59 } progstate_s;
60
61 static progstate_s progstate;
62
63 static int isdir(stringptr* file) {
64 return file->size && file->ptr[file->size -1] == '/';
65 }
66
67 static void copyDate(stringptr* file, struct stat* st) {
68 struct utimbuf ut;
69 ut.modtime = st->st_mtime;
70 ut.actime = st->st_atime;
71 if(utime(file->ptr, &ut) == -1)
72 log_perror("utime");
73 }
74
78b1feb rofl0r multiple fixes
authored
75 static void restoreTrailingSlash(stringptr* s) {
76 s->ptr[s->size] = '/';
77 s->size++;
78 s->ptr[s->size] = 0;
79 }
80
81 static inline int removeTrailingSlash(stringptr* s) {
82 if(isdir(s)) {
83 stringptr_shiftleft(s, 1);
84 return 1;
65dc42b rofl0r initial commit
authored
85 }
78b1feb rofl0r multiple fixes
authored
86 return 0;
87 }
88
89 static void updateTimestamp(stringptr* dst, struct stat* ss) {
90 int wasdir = removeTrailingSlash(dst);
91 copyDate(dst, ss);
92 if(wasdir) restoreTrailingSlash(dst);
93 }
94
95 static void makeDir(stringptr* dst, struct stat* ss) {
96 if(progstate.simulate)
97 return;
98 if(mkdir(dst->ptr, ss->st_mode) == -1) {
65dc42b rofl0r initial commit
authored
99 log_perror("mkdir");
100 return;
101 }
102 }
103
104 static char* getMbsString(char* mbs_buf, size_t buf_size, uint64_t bytes, long ms) {
105 float mbs = bytes ?
106 (((float) bytes / (1024.f * 1024.f)) /
107 ((float) ms / 1000.f)) :
108 0.f;
109 unsigned mbs_a = (unsigned) mbs;
110 unsigned mbs_b = (unsigned)((mbs - mbs_a) * 100.f);
111 ulz_snprintf(mbs_buf, buf_size, "%u.%.2u MB/s", mbs_a, mbs_b);
112 return mbs_buf;
113 }
114
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
115 static void doSync(stringptr* src, stringptr* dst, struct stat *src_stat, char* reason) {
65dc42b rofl0r initial commit
authored
116 int fds, fdd;
117
118 int errclose;
119 stringptr* err_data;
120 char* err_func;
121
122 uint64_t done = 0;
123 CRC32C_CTX crc;
124 union {
125 uint32_t asInt;
126 uint8_t asChar[4];
127 } crc_result;
128
129 struct timeval starttime;
130 char buf[src_stat->st_blksize];
78b1feb rofl0r multiple fixes
authored
131 long time_passed;
132
133 if(progstate.simulate) {
134 crc_result.asInt = 0;
135 time_passed = 0;
136 goto stats;
137 }
65dc42b rofl0r initial commit
authored
138
139 if((fds = open(src->ptr, O_RDONLY)) == -1) {
140 err_data = src;
141 err_func = "open";
142 errclose = 0;
143
144 error:
145
146 log_puts(2, err_data);
147 log_puts(2, SPL(" "));
148 log_perror(err_func);
149 if(errclose) {
150 close(fds);
151 errclose--;
152 if(errclose) {
153 close(fdd);
154 errclose--;
155 }
156 }
157 return;
158 };
159 if((fdd = open(dst->ptr, O_WRONLY | O_CREAT | O_TRUNC, src_stat->st_mode)) == -1) {
160 err_data = dst;
161 errclose = 1;
162 err_func = "open";
163 goto error;
164 };
78b1feb rofl0r multiple fixes
authored
165
166 // we always compute the CRC, because it's nearly "for free",
167 // since the file has to be read anyway.
65dc42b rofl0r initial commit
authored
168 CRC32C_Init(&crc);
169 gettimestamp(&starttime);
170 while(done < (uint64_t) src_stat->st_size) {
171 ssize_t nread = read(fds, buf, src_stat->st_blksize);
172 if(nread == -1) {
173 err_data = src;
174 err_func = "read";
175 errclose = 2;
176 goto error;
177 } else if (nread == 0)
178 break;
179 else {
180 ssize_t nwrote = 0, nwrote_s;
181
182 CRC32C_Update(&crc, (const uint8_t*) buf, nread);
183
184 while(nwrote < nread) {
185 nwrote_s = write(fdd, buf + nwrote, nread - nwrote);
186 if(nwrote_s == -1) {
187 err_data = dst;
188 errclose = 2;
189 err_func = "write";
190 goto error;
191 }
192 nwrote += nwrote_s;
193 }
194 done += nread;
195 }
196 }
197 close(fds);
198 close(fdd);
199 copyDate(dst, src_stat);
200 char crc_str[16];
201 char mbs_str[64];
202 CRC32C_Final(crc_result.asChar, &crc);
78b1feb rofl0r multiple fixes
authored
203
204 time_passed = mspassed(&starttime);
205
206 stats:
8ccc041 rofl0r print crc in network order
authored
207 ulz_snprintf(crc_str, sizeof(crc_str), "%.8x", htonl(crc_result.asInt));
65dc42b rofl0r initial commit
authored
208
209 // we do not use printf because it has a limited buffer size
210 log_put(1, VARISL("CRC: "), VARIC(crc_str), VARISL(", "),
211 VARIS(src), VARISL(" -> "), VARIS(dst),
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
212 VARISL(" @"), VARIC(getMbsString(mbs_str, sizeof(mbs_str), src_stat->st_size, time_passed)),
213 VARISL(" ("), VARIC(reason), VARISL(")"),
214 NULL);
65dc42b rofl0r initial commit
authored
215
216 progstate.total.copied += src_stat->st_size;
217 progstate.total.copies += 1;
218 }
219
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
220 static void doFile(stringptr* src, stringptr* dst, stringptr* diff, struct stat* ss) {
65dc42b rofl0r initial commit
authored
221 struct stat sd;
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
222 char* reason = "x";
65dc42b rofl0r initial commit
authored
223 if(stat(dst->ptr, &sd) == -1) {
224 switch(errno) {
225 case ENOENT:
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
226 if(progstate.checkExists) {
227 reason = "e";
228 do_sync:
229 doSync(src, diff, ss, reason);
230 }
65dc42b rofl0r initial commit
authored
231 return;
232 default:
233 log_puts(2, dst);
234 log_puts(2, SPL(" "));
235 log_perror("stat dest");
236 break;
237 }
238 }
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
239 if(progstate.checkFileSize && ss->st_size != sd.st_size) {
240 reason = "f";
241 goto do_sync;
242 } else if (progstate.checkDate && ss->st_mtime > sd.st_mtime) {
243 reason = "d";
244 goto do_sync;
245 } else if (progstate.checkDateOlder && ss->st_mtime < sd.st_mtime) {
246 reason = "o";
247 goto do_sync;
248 } else if (!progstate.checkDateOlder && ss->st_mtime < sd.st_mtime) {
65dc42b rofl0r initial commit
authored
249 ulz_fprintf(2, "dest is newer than source: %s , %s : %llu , %llu\n", src->ptr, dst->ptr, ss->st_mtime, sd.st_mtime);
250 } else if(progstate.checkChecksum) {
251 /* TODO launch 2 processes, each computing the CRC of src/dest in parallel,
252 * then join em and compare crc and warn and copy if different
253 */
254 }
255 progstate.total.skipped += 1;
256 }
257
78b1feb rofl0r multiple fixes
authored
258 static void setLinkTimestamp(stringptr* link, struct stat* ss) {
259 struct timeval tv[2];
260 tv[0].tv_sec = ss->st_atime;
261 tv[0].tv_usec = ss->st_atim.tv_nsec / 1000;
262 tv[1].tv_sec = ss->st_mtime;
263 tv[1].tv_usec = ss->st_mtim.tv_nsec / 1000;
264 if(lutimes(link->ptr, tv) == -1) {
265 log_perror("lutimes");
266 }
65dc42b rofl0r initial commit
authored
267 }
268
269 // FIXME dont copy symlink if the target is equal
270 // FIXME dont increment progstate.total.symlink in case of failure
271 static void doLink(stringptr* src, stringptr* dst, struct stat* ss) {
272 char buf[4096 + 1];
273 int wasdir = 0;
274 struct stat sd;
78b1feb rofl0r multiple fixes
authored
275 ssize_t ret;
276 if(progstate.simulate) goto skip;
277
278 ret = readlink(src->ptr, buf, sizeof(buf) - 1);
65dc42b rofl0r initial commit
authored
279 if(ret == -1) {
280 log_puts(2, src);
281 log_puts(2, SPL(" "));
282 log_perror("readlink");
283 return;
284 } else if (!ret) {
285 log_puts(2, src);
286 log_puts(2, SPL(" "));
287 log_puts(2, SPL("readlink returned 0"));
288 return;
289 }
290 buf[ret] = 0;
291
78b1feb rofl0r multiple fixes
authored
292 wasdir = removeTrailingSlash(dst);
65dc42b rofl0r initial commit
authored
293
294 if(!(lstat(dst->ptr, &sd) == -1 && errno == ENOENT)) {
295 //dst already exists, we need to unlink it for symlink to succeed
296 //if(S_ISLNK(sd.st_mode))
297 if(unlink(dst->ptr) == -1) {
298 log_puts(2, dst);
299 log_puts(2, SPL(" "));
300 log_perror("unlink");
301 }
302 }
303
304 if(symlink(buf, dst->ptr) == -1) {
305 log_putc(2, buf);
306 log_puts(2, SPL(" -> "));
307 log_puts(2, dst);
308 log_puts(2, SPL(" "));
309 log_perror("symlink");
310 } else {
311 log_putc(1, buf);
312 log_puts(1, SPL(" >> "));
313 log_puts(1, dst);
314 log_putln(1);
315 }
316
78b1feb rofl0r multiple fixes
authored
317 setLinkTimestamp(dst, ss);
65dc42b rofl0r initial commit
authored
318
78b1feb rofl0r multiple fixes
authored
319 if(wasdir)
65dc42b rofl0r initial commit
authored
320 restoreTrailingSlash(dst);
78b1feb rofl0r multiple fixes
authored
321 skip:
65dc42b rofl0r initial commit
authored
322 progstate.total.symlink += 1;
323 }
324
325 static void doDir(stringptr* subd) {
326 filelist f;
327 stringptr *combined_src = stringptr_concat(progstate.srcdir, subd, NULL);
328 stringptr *combined_dst = stringptr_concat(progstate.dstdir, subd, NULL);
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
329 stringptr *combined_diff = stringptr_concat(progstate.diffdir, subd, NULL);
330
65dc42b rofl0r initial commit
authored
331 struct stat src_stat;
332
333 if(!filelist_search(&f, combined_src, SPL("*"), FLF_EXCLUDE_PATH | FLF_INCLUDE_HIDDEN)) {
334 stringptr* file;
335 stringptr* file_combined_src;
336 stringptr* file_combined_dst;
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
337 stringptr* file_combined_diff;
65dc42b rofl0r initial commit
authored
338 sblist_iter(f.files, file) {
339 file_combined_src = stringptr_concat(combined_src, file, NULL);
340 file_combined_dst = stringptr_concat(combined_dst, file, NULL);
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
341 file_combined_diff = stringptr_concat(combined_diff, file, NULL);
78b1feb rofl0r multiple fixes
authored
342
343 removeTrailingSlash(file_combined_src); // remove trailing slash so stat doesnt resolve symlinks...
344
65dc42b rofl0r initial commit
authored
345 if(lstat(file_combined_src->ptr, &src_stat) == -1) {
346 log_puts(2, file_combined_src);
347 log_puts(2, SPL(" "));
348 log_perror("stat");
349 } else {
350 if(S_ISLNK(src_stat.st_mode)) {
351
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
352 doLink(file_combined_src, file_combined_diff, &src_stat);
65dc42b rofl0r initial commit
authored
353
354 } else if(isdir(file)) {
355 restoreTrailingSlash(file_combined_src);
356
357 stringptr *path_combined = stringptr_concat(subd, file, NULL);
057e220 rofl0r simulate: dont create directories and try to copyDate()
authored
358 if(!progstate.simulate && access(file_combined_diff->ptr, R_OK) == -1 && errno == ENOENT) {
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
359 makeDir(file_combined_diff, &src_stat);
78b1feb rofl0r multiple fixes
authored
360 }
361 // else updateTimestamp(file_combined_dst, &src_stat);
65dc42b rofl0r initial commit
authored
362 doDir(path_combined);
363 stringptr_free(path_combined);
057e220 rofl0r simulate: dont create directories and try to copyDate()
authored
364 if(!progstate.simulate)
365 updateTimestamp(file_combined_diff, &src_stat);
65dc42b rofl0r initial commit
authored
366 } else {
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
367 doFile(file_combined_src, file_combined_dst, file_combined_diff, &src_stat);
65dc42b rofl0r initial commit
authored
368 }
c24c7e2 rofl0r increase counter on skipifnewer
authored
369 }
65dc42b rofl0r initial commit
authored
370 stringptr_free(file_combined_src);
371 stringptr_free(file_combined_dst);
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
372 stringptr_free(file_combined_diff);
65dc42b rofl0r initial commit
authored
373 }
374 filelist_free(&f);
375 }
376
377 stringptr_free(combined_src);
378 stringptr_free(combined_dst);
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
379 stringptr_free(combined_diff);
65dc42b rofl0r initial commit
authored
380 }
381
382 static void printStats(long ms) {
383 char mbs_buf[64];
384 ulz_fprintf(1, "copied: %llu\n"
385 "skipped: %llu\n"
386 "symlinks: %llu\n"
387 "bytes copied: %llu\n"
388 "seconds: %lu\n"
389 "rate: %s\n",
390 progstate.total.copies,
391 progstate.total.skipped,
392 progstate.total.symlink,
393 progstate.total.copied,
394 ms / 1000,
395 getMbsString(mbs_buf, sizeof(mbs_buf), progstate.total.copied, ms)
396 );
397 }
398
399 static int syntax() {
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
400 log_puts(1, SPL("filesync OPTIONS srcdir dstdir [diffdir]\n\n"
401 "if diffdir is given, the program will check for files in destdir,\n"
402 "but will write into diffdir instead. this allows usage as a simple\n"
403 "incremental backup tool.\n\n"
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
404 "\toptions: -s[imulate] -e[xists] -d[ate] -o[lder] -f[ilesize] -c[hecksum]\n"
78b1feb rofl0r multiple fixes
authored
405 "\t-s : only simulate and print to stdout (dry run)\n"
057e220 rofl0r simulate: dont create directories and try to copyDate()
authored
406 "\t note: will not print symlinks currently\n"
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
407 "\t-e : copy source files that dont exist on the dest side\n"
408 "\t-d : copy source files with newer timestamp (modtime)\n"
409 "\t-o : copy source files with older timestamp (modtime)\n"
410 "\t-f : copy source files with different filesize\n"
411 "\t-c : copy source files if checksum is different (not implemented yet)\n\n"
78b1feb rofl0r multiple fixes
authored
412 "WARNING: you should *always* redirect stdout and stderr\n"
413 "into some logfile. to see the actual state, attach with\n"
414 "tail -f or tee...\n"
415 "After a full run you can pipe the stdout.txt into the supplied\n"
416 "perl script which can check the CRCs, in case you want to\n"
417 "verify the copy. it is proposed that this run happens separately,\n"
418 "so that the copied files are no longer buffered.\n\n"
419 ));
65dc42b rofl0r initial commit
authored
420 return 1;
421 }
422
423 int main (int argc, char** argv) {
78b1feb rofl0r multiple fixes
authored
424
425 if(argc < 4) return syntax();
65dc42b rofl0r initial commit
authored
426 int startarg = 1;
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
427 int freedst = 0, freediff = 0;
428 int dirargs = 0, i;
65dc42b rofl0r initial commit
authored
429 struct timeval starttime;
78b1feb rofl0r multiple fixes
authored
430 struct stat src_stat;
431
432 op_state op_b, *op = &op_b;
65dc42b rofl0r initial commit
authored
433
78b1feb rofl0r multiple fixes
authored
434 op_init(op, argc, argv);
435
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
436 progstate.simulate = op_hasflag(op, SPL("s")) || op_hasflag(op, SPL("simulate"));
78b1feb rofl0r multiple fixes
authored
437
ce49204 rofl0r add (o)lder option, and remove skipifnewer
authored
438 progstate.checkExists = op_hasflag(op, SPL("e")) || op_hasflag(op, SPL("exists"));
439 progstate.checkFileSize = op_hasflag(op, SPL("f")) || op_hasflag(op, SPL("filesize"));
440 progstate.checkDate = op_hasflag(op, SPL("d")) || op_hasflag(op, SPL("date"));
441 progstate.checkDateOlder = op_hasflag(op, SPL("o")) || op_hasflag(op, SPL("older"));
442 progstate.checkChecksum = op_hasflag(op, SPL("c")) || op_hasflag(op, SPL("checksum"));
78b1feb rofl0r multiple fixes
authored
443
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
444 for(i = 1; i < argc; i++)
445 if(argv[i][0] != '-') dirargs++;
446
447 if(dirargs < 2 || dirargs > 3) {
448 log_puts(2, SPL("invalid arguments detected\n"));
e0ea856 rofl0r react sanely on permission errors and the like
authored
449 return syntax();
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
450 }
451
452 startarg = argc - dirargs;
65dc42b rofl0r initial commit
authored
453
454 memset(&progstate.total, 0, sizeof(totals));
455
456 progstate.srcdir = stringptr_fromchar(argv[startarg], &progstate.srcdir_b);
457 progstate.dstdir = stringptr_fromchar(argv[startarg+1], &progstate.dstdir_b);
5f6e6db rofl0r simplify statement
authored
458 progstate.diffdir = stringptr_fromchar((dirargs == 3) ? argv[startarg+2] : argv[startarg+1], &progstate.diffdir_b);
65dc42b rofl0r initial commit
authored
459
e0ea856 rofl0r react sanely on permission errors and the like
authored
460 if(access(progstate.diffdir->ptr, R_OK) == -1) {
461 if(errno == ENOENT) {
462 if(stat(progstate.srcdir->ptr, &src_stat) == -1) {
463 log_perror("stat");
464 return 1;
465 }
466 makeDir(progstate.diffdir, &src_stat);
467 } else {
468 log_perror("uncaught error while trying to access dest/diff dir");
78b1feb rofl0r multiple fixes
authored
469 return 1;
470 }
471 }
472
65dc42b rofl0r initial commit
authored
473 if(!isdir(progstate.dstdir)) {
474 progstate.dstdir = stringptr_concat(progstate.dstdir, SPL("/"), NULL);
475 freedst = 1;
476 }
477
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
478 if(!isdir(progstate.diffdir)) {
479 progstate.diffdir = stringptr_concat(progstate.diffdir, SPL("/"), NULL);
480 freediff = 1;
481 }
482
65dc42b rofl0r initial commit
authored
483 gettimestamp(&starttime);
484
485 CRC32C_InitTables();
486
487 doDir(isdir(progstate.srcdir) ? SPL("") : SPL("/"));
488
489 printStats(mspassed(&starttime));
490
491 if(freedst) stringptr_free(progstate.dstdir);
73d930d rofl0r implemented usage of a 'diffdir', which receives the changed files
authored
492 if(freediff) stringptr_free(progstate.diffdir);
65dc42b rofl0r initial commit
authored
493
494 return 0;
495 }
Something went wrong with that request. Please try again.