Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 364 lines (326 sloc) 11.569 kb
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
1 #!/usr/bin/perl
2
3 # sidmirror.pl - Sid mirroring script
369a13f6 »
2007-03-10 * added TODO file
4 # (c) 2003-2007 Alex Malinovich (demonbane@the-love-shack.net)
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
5 # Released under the GPL
6 # See www.fsf.org for a full copy of the GPL.
7 #
8 # $server - Server to sync main, contrib, and non-free with.
9 #
10 # $serverpath - Path to the debian directory on $server. This is
11 # USUALLY /debian. Some mirrors, such as rsync.kernel.org, use a
12 # different path, such as /mirrors/debian.
13 #
14 # $rootdir - root directory for main, contrib, and non-free. This will
15 # become identical to $server::debian/ after the first rsync.
16 #
17 # $installpath - directory where this script resides. Since all of the
18 # paths used for logs and associated programs are relative, we need to
19 # chdir to the home directory first to make sure that nothing breaks
20 # if this is being run from a cron job.
21 #
22 # $arch - the architecture to copy. This has been tested with i386 and
23 # amd64. All other architectures SHOULD work but they have not been
24 # tested.
25
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
26 $version="0.11";
369a13f6 »
2007-03-10 * added TODO file
27
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
28 use Term::ProgressBar 2.00;
29 use Fcntl;
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
30 use Config::File;
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
31
dea33197 »
2007-10-23 * Ok, so I didn't really do a good job with the last version. This still
32 $autorun = 0;
33 $force = 0;
e9f57815 »
2007-10-23 * Added -n option to allow running without a config file.
34 $noconfig = 0;
dea33197 »
2007-10-23 * Ok, so I didn't really do a good job with the last version. This still
35
36 while (@ARGV) {
37 my $option = shift (@ARGV);
38 if ($option eq "--help" || $option eq "-h") {
39 print "Usage: sidmirror [OPTION]\n";
40 print "Update local Debian Sid mirror.\n\n";
41 print " -h, --help display this screen and exit\n";
42 print " -a, --auto run in automated mode; no user intervention required\n";
e9f57815 »
2007-10-23 * Added -n option to allow running without a config file.
43 print " -n, --noconfig run even if there is no config file (uses defaults)\n";
dea33197 »
2007-10-23 * Ok, so I didn't really do a good job with the last version. This still
44 print " -f, --force ignore lockfile and run anyway (DANGEROUS!)\n";
4152a42d »
2008-09-28 Adding --quiet option for crontabs
45 print " -q, --quiet only print errors (implies -a)\n";
dea33197 »
2007-10-23 * Ok, so I didn't really do a good job with the last version. This still
46 print " -v, --version display version and exit\n\n";
47 exit 0;
48 }elsif ($option eq "-a" || $option eq "--auto") {
49 $autorun = 1;
50 }elsif ($option eq "-f" || $option eq "--force") {
51 $force = 1;
52 }elsif ($option eq "-v" || $option eq "--version") {
53 print "sidmirror $version\n";
54 exit 0;
e9f57815 »
2007-10-23 * Added -n option to allow running without a config file.
55 }elsif ($option eq "-n" || $option eq "--noconfig") {
56 $noconfig = 1;
4152a42d »
2008-09-28 Adding --quiet option for crontabs
57 }elsif ($option eq "-q" || $option eq "--quiet") {
58 $quiet = 1;
59 $autorun = 1;
dea33197 »
2007-10-23 * Ok, so I didn't really do a good job with the last version. This still
60 }elsif ($option =~ /^\w/) {
61 $server = $option;
62 }else {
63 print STDERR "sidmirror: invalid option -- $option\n";
64 print STDERR "Try \`sidmirror --help\' for more information.\n";
65 exit 1;
66 }
67 }
68
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
69 if (-e "/etc/sidmirror.conf") {
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
70 my $config_hash = Config::File::read_config_file("/etc/sidmirror.conf");
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
71
72 if ($config_hash->{"Mirror"}) {
73 $server = $config_hash->{"Mirror"};
74 }else {
75 $server = "http.us.debian.org";
76 }
77
78 if ($config_hash->{"RepositoryPath"}) {
79 $serverpath = $config_hash->{"RepositoryPath"};
80 if (substr($serverpath, 0, 1) eq "/") {
81 $serverpath=substr($serverpath, 1);
4152a42d »
2008-09-28 Adding --quiet option for crontabs
82 print "Modifying RepositoryPath to \"$serverpath\"\n" unless ($quiet);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
83 }
84 }else {
85 $serverpath = "debian";
86 }
87
88 if ($config_hash->{"LocalPath"}) {
89 $rootdir = $config_hash->{"LocalPath"};
90 }else {
91 $rootdir = "./debian";
92 }
93
94 if ($config_hash->{"Architecture"}) {
369a13f6 »
2007-03-10 * added TODO file
95 @arch = split(/\,/, $config_hash->{"Architecture"});
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
96 }else {
369a13f6 »
2007-03-10 * added TODO file
97 $arch[0] = "i386";
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
98 }
99
100 if ($config_hash->{"LogDir"}) {
101 $logdir = $config_hash->{"LogDir"};
102 }else {
103 $logdir = "./logs";
104 }
105
106 if ($config_hash->{"InstallPath"}) {
107 $installpath = $config_hash->{"InstallPath"};
108 }else{
109 $installpath = "./";
110 }
e9f57815 »
2007-10-23 * Added -n option to allow running without a config file.
111 }elsif (!$noconfig) {
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
112 print "sidmirror configuration file (sidmirror.conf) not found!\n";
e9f57815 »
2007-10-23 * Added -n option to allow running without a config file.
113 print "Please create an appropriate configuration file and try again or run\nwith the -n option to use defaults\n\n";
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
114 exit 0;
115 }
116
117 chdir $installpath || die ("Unable to enter InstallPath!\n");
118
119 ###############################################################
120 ## Create a lockfile to prevent multiple instances from running
121 ###############################################################
122 if (!sysopen(FH, "sidmirror.lock", O_WRONLY|O_EXCL|O_CREAT, 0400)) {
123 if (-M "sidmirror.lock" >= 1.5) {
124 print "WARNING!!!\nStale lockfile found. Removing... ";
125 do {
126 $pid = `pidof -x -o %PPID sidmirror`;
96a4f088 »
2008-07-26 smarter pid checking
127 if ($pid ne "\n" && $pid) {`kill -9 $pid`};
128 }while ($pid ne "\n" && $pid);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
129 unlink "sidmirror.lock";
96a4f088 »
2008-07-26 smarter pid checking
130 print "Done!\n";
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
131 sysopen(FH, "sidmirror.lock", O_WRONLY|O_EXCL|O_CREAT, 0400) || die $!;
132 }elsif ($force) {
133 print "Lockfile found, but proceeding anyway as you request.\n";
134 print "!!! DANGER !!!\n";
135 print "This may break your mirror! You have been warned!\n\n";
136 }else {
137 print "Lockfile found, aborting!\n";
4152a42d »
2008-09-28 Adding --quiet option for crontabs
138 exit 1;
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
139 }
140 }
141
142 ################################################
143 ## Download the newest Packages.gz from $server
144 ################################################
145 do {
4152a42d »
2008-09-28 Adding --quiet option for crontabs
146 print "Retrieving newest Packages.gz from $server...\n" unless ($quiet);
369a13f6 »
2007-03-10 * added TODO file
147 # Generate our include string
148 foreach $includearch (@arch) {
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
149 push(@packageincludes,"/main/binary-$includearch/Packages.gz",
150 "/contrib/binary-$includearch/Packages.gz",
151 "/non-free/binary-$includearch/Packages.gz",
152 "/main/debian-installer/binary-$includearch/Packages.gz"
153 );
154 }
155 foreach $packageinclude (@packageincludes) {
156 $includestring .= " --include \"$packageinclude\"";
369a13f6 »
2007-03-10 * added TODO file
157 }
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
158 $errcode = system(
369a13f6 »
2007-03-10 * added TODO file
159 "rsync -P --recursive --times --verbose --compress --delete-excluded".$includestring." --include \"*/\" --exclude \"*\" $server\:\:$serverpath/dists/sid/ $rootdir/dists/sid/ 1> Package.output 2> Package.error"
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
160 );
161 $errcode /= 256;
162 $retry = "false";
163 if ($errcode > 0 && $autorun != 1) {
164 print "\nError! Rsync failed with an exit code of $errcode! Check Package.error for details.\n<A>bort, <R>etry, or <C>ontinue? [A/r/c] ";
165 $proceed = <STDIN>;
166 $proceed = uc($proceed);
167 chomp $proceed;
168 if ($proceed eq "C") {
169 $retry = "false";
170 }elsif ($proceed eq "R") {
171 $retry = "true";
172 }else {
173 unlink ("sidmirror.lock");
174 exit 1;
175 }
4152a42d »
2008-09-28 Adding --quiet option for crontabs
176 }elsif (! $quiet) {
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
177 print "\nDone!\n\n";
178 }
179 }while ($retry eq "true");
180
181 # TODO: Either make this more failsafe or get rid of it and
182 # use a user-defined array.
183 opendir (DISTDIR, "$rootdir/dists/sid/");
184 @dirlist = grep {!/^\./ && -d "$rootdir/dists/sid/$_"} readdir (DISTDIR);
185 closedir DISTDIR;
186
187 # Dirty hack, but necessary to include debian installer.
188 push (@dirlist, "main/debian-installer");
189
369a13f6 »
2007-03-10 * added TODO file
190 foreach $packagearch (@arch) {
191 foreach (@dirlist) {
192 my $packname = "$rootdir/dists/sid/$_/binary-$packagearch/Packages.gz";
193 if (-s $packname) {
4152a42d »
2008-09-28 Adding --quiet option for crontabs
194 print "Reading records from ($packagearch) $_ Packages.gz...\n" unless ($quiet);
369a13f6 »
2007-03-10 * added TODO file
195 $oldcount = $#packfile;
196 push (@packfile, `gunzip --to-stdout $packname`);
4152a42d »
2008-09-28 Adding --quiet option for crontabs
197 print (($#packfile + 1 - $oldcount), " records read from $_\n\n") unless ($quiet);
369a13f6 »
2007-03-10 * added TODO file
198 push (@modch, "$packname");
199 }
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
200 }
201 }
202
203 undef $flagvar;
204 foreach (@packfile) {
205 if ($flagvar) {
206 $filenames{$flagvar} = substr ($_, 6);
207 undef $flagvar;
208 }
209 if ($_ =~ /^Filename\: /) {
210 $flagvar = substr($_, 10);
211 chomp $flagvar;
212 }
213 }
214
4152a42d »
2008-09-28 Adding --quiet option for crontabs
215 print "Done!\n\n" unless ($quiet);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
216
217 # TODO: _POSSIBLY_ have the script compare the files already present to
218 # the names and sizes listed in Packages.gz and exclude ones that match.
219 # This should speed things up quite a bit when there's a small number of
220 # packages to be updated. I'll have to think about this one.
221
222 # Generate a list of files we need from the standard distrib
4152a42d »
2008-09-28 Adding --quiet option for crontabs
223 print "Generating includefile...\n" unless ($quiet);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
224
225 open (INCLUDEFILE, ">", "includefile");
226
227 if (-e "static-includes") {
228 open (SINCLUDES, "static-includes");
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
229 while (<SINCLUDES>) {
230 if ($_ =~ /^\w{3,6}\:\/\/.*\,/) {
231 push(@urlincludes, $_);
232 }else {
233 push(@sincludes, $_);
234 }
235 }
4152a42d »
2008-09-28 Adding --quiet option for crontabs
236 if (! $quiet) {
237 print "Inserting ", ($#sincludes + 1), " records from static-includes...\n";
238 print "Inserting ", ($#urlincludes + 1), " URL's from static-includes...\n";
239 }
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
240 close SINCLUDES;
241 print INCLUDEFILE @sincludes;
242 }
243
244 if ($autorun != 1) {
245 print "Number of files: ",(scalar keys %filenames),"\n";
246 $progressbar = Term::ProgressBar->new(scalar keys %filenames);
247 }
248
249 $includedfiles = 0;
250 $currentcount = 0;
251 foreach (keys %filenames) {
252 if ($autorun != 1) {
253 $currentcount++;
254 if ($currentcount >= $next_update) {
255 $next_update = $progressbar->update($currentcount);
256 }
257 }
258 if (!-e "$rootdir/$_") {
259 print INCLUDEFILE $_."\n";
260 $includedfiles++;
261 $sizetoget += $filenames{$_};
262 }
263 }
264
265 if ($currentcount >= $next_update && $autorun != 1) {
266 $next_update = $progressbar->update($currentcount);
267 }
268
269 close INCLUDEFILE;
4152a42d »
2008-09-28 Adding --quiet option for crontabs
270 if (! $quiet) {
271 print $includedfiles, " records written\n";
272 printf ("%.2f", ($sizetoget / 1048576));
273 print "MB to download\n";
274 }
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
275 $ussize = $sizetoget;
276
277 if ($autorun != 1) {
278 print "Run cleanup scripts now? [Y/n] ";
279 $proceed = <STDIN>;
280 $proceed = uc($proceed);
281 chomp $proceed;
282 }
283 if ($proceed ne "N") {
4152a42d »
2008-09-28 Adding --quiet option for crontabs
284 my $dupeargs = "";
285 my $parseargs = " -h";
286 my $cleanargs = "";
287 if ($autorun) {
288 $dupeargs = " -a";
289 }
290 if ($quiet) {
291 $dupeargs .= " -q";
292 $parseargs = " -q";
293 $cleanargs = " -q";
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
294 }
4152a42d »
2008-09-28 Adding --quiet option for crontabs
295 system ("./cleanup/dupesearch.pl".$dupeargs);
296 system ("./cleanup/parseoldsize.pl".$parseargs);
297 system ("./cleanup/cleanold.pl".$cleanargs);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
298 }
299
300 if ($autorun != 1) {
301 print "\nReady to begin mirroring operation.\n";
302 print "Need to download ";
303 printf ("%.2f", $ussize / 1048576);
304 print "MB of packages. Continue? [Y/n] ";
305 $proceed = <STDIN>;
306 $proceed = uc($proceed);
307 chomp $proceed;
308 }
309
310 chmod 0555, @modch;
311
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
312 if (-s "./includefile" && $proceed ne "N") {
4152a42d »
2008-09-28 Adding --quiet option for crontabs
313 print "Starting rsync with $server...\n" unless ($quiet);
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
314 `echo Starting rsync with $server... > $logdir/rsync.log`;
315 $retcode = system ("rsync --recursive --links --hard-links --times --verbose --compress --include \"*/\" --include-from=includefile --exclude \"*\" $server\:\:$serverpath/ $rootdir/ >> $logdir/rsync.log 2>&1");
316 $retcode /= 256;
317 `echo End rsync with $server... exit value = $retcode >> $logdir/rsync.log`;
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
318 if ($retcode > 0) {
319 print "An error was encountered while performing the rsync operation! The exit code reported was $retcode. Please check $logdir/rsync.log.0 for details.\n\n";
4152a42d »
2008-09-28 Adding --quiet option for crontabs
320 }elsif (! $quiet) {
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
321 print "rsync completed successfully!\n\n";
322 }
4152a42d »
2008-09-28 Adding --quiet option for crontabs
323 }elsif (! $quiet) {
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
324 print "No files to fetch from $server... skipping...\n";
325 `echo No files to fetch from $server... skipping... >> $logdir/rsync.log`;
326 }
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
327
328 undef($retcode);
329
330 if (@urlincludes) {
331 for (@urlincludes) {
332 ($myurl, $mypath, $mycommand) = split(/,/);
4152a42d »
2008-09-28 Adding --quiet option for crontabs
333 print "Starting wget with $myurl...\n" unless ($quiet);
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
334 $retcode = system("wget -a $logdir/wget.log -N -P $rootdir/$mypath $myurl");
f0de546d »
2007-11-02 * Fixed return value issue with system call to wget.
335 $retcode /= 256;
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
336 if ($retcode > 0) {
337 print "An error was encountered while performing the wget operation with $myurl! The exit code reported was $retcode. Please check $logdir/wget.log.0 for details.\n\n";
c5d69baf »
2009-02-05 Commands in static-includes now get executed properly
338 }else {
339 print "wget with $myurl completed successfully!\n" unless ($quiet);
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
340 if ($mycommand) {
341 chomp($mycommand);
342 chdir "$rootdir/$mypath" || die ("Unable to enter $rootdir/$mypath!\n");
4152a42d »
2008-09-28 Adding --quiet option for crontabs
343 print "Executing '$mycommand'...\n\n" unless ($quiet);
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
344 `$mycommand`;
345 chdir $installpath || die ("Unable to enter InstallPath!\n");
4152a42d »
2008-09-28 Adding --quiet option for crontabs
346 }elsif (! $quiet) {
ae04617e »
2007-10-23 * Initial setup of Debian package. Still have a few kinks to work out…
347 print "\n";
348 }
349 }
350 }
351 `savelog $logdir/wget.log`;
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
352 }
353
354 `savelog $logdir/rsync.log`;
355
356 chmod 0755, @modch;
357
c579db5f »
2008-10-08 Better fix for error message printing
358 if ($retcode == 0 && $errcode == 0) {
359 print "All operations completed successfully!\n\n" unless ($quiet);
360 }else {
28ffe121 »
2007-03-10 * Initial commit at version 0.9c
361 print "Errors encountered during operation! Please check Package.error and $logdir/rsync.log.0 for details!\n\n";
362 }
363
364 unlink "sidmirror.lock";
Something went wrong with that request. Please try again.