scripts/download.pl: detach mirror URLs from script file

Detach URL from download script to make it easier to change and update the URLs. The mirror list is moved to a JSON file in the scripts directory called projectsmirrors.json. Link: openwrt#15745 Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
Ansuel · Jun 18, 2024 · 465cf35 · 465cf35
1 parent fcdc7da
commit 465cf35
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 50 deletions.
diff --git a/scripts/download.pl b/scripts/download.pl
@@ -12,6 +12,7 @@
 use File::Basename;
 use File::Copy;
 use Text::ParseWords;
+use JSON::PP;
 
 @ARGV > 2 or die "Syntax: $0 <target dir> <filename> <hash> <url filename> [<mirror> ...]\n";
 
@@ -56,6 +57,20 @@ sub localmirrors {
 	return @mlist;
 }
 
+sub projectsmirrors {
+	my $project = shift;
+
+	open (PM, "$scriptdir/projectsmirrors.json") ||
+		die "Can´t open $scriptdir/projectsmirrors.json: $!\n";
+	local $/;
+	my $mirror_json = <PM>;
+	my $mirror = decode_json $mirror_json;
+
+	foreach (@{$mirror->{$project}}) {
+		push @mirrors, $_ . "/" . ($1 or "");
+	}
+}
+
 sub which($) {
 	my $prog = shift;
 	my $res = `command -v $prog`;
@@ -244,51 +259,23 @@ sub cleanup
 	if ($mirror =~ /^\@SF\/(.+)$/) {
 		# give sourceforge a few more tries, because it redirects to different mirrors
 		for (1 .. 5) {
-			push @mirrors, "https://downloads.sourceforge.net/$1";
+			projectsmirrors '@SF', $1;
 		}
 	} elsif ($mirror =~ /^\@OPENWRT$/) {
 		# use OpenWrt source server directly
 	} elsif ($mirror =~ /^\@DEBIAN\/(.+)$/) {
-		push @mirrors, "https://ftp.debian.org/debian/$1";
-		push @mirrors, "https://mirror.leaseweb.com/debian/$1";
-		push @mirrors, "https://mirror.netcologne.de/debian/$1";
-		push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/debian/$1";
-		push @mirrors, "https://mirrors.ustc.edu.cn/debian/$1"
+		projectsmirrors '@DEBIAN', $1;
 	} elsif ($mirror =~ /^\@APACHE\/(.+)$/) {
-		push @mirrors, "https://dlcdn.apache.org/$1";
-		push @mirrors, "https://mirror.aarnet.edu.au/pub/apache/$1";
-		push @mirrors, "https://mirror.csclub.uwaterloo.ca/apache/$1";
-		push @mirrors, "https://archive.apache.org/dist/$1";
-		push @mirrors, "https://mirror.cogentco.com/pub/apache/$1";
-		push @mirrors, "https://mirror.navercorp.com/apache/$1";
-		push @mirrors, "https://ftp.jaist.ac.jp/pub/apache/$1";
-		push @mirrors, "https://apache.cs.utah.edu/apache.org/$1";
-		push @mirrors, "http://apache.mirrors.ovh.net/ftp.apache.org/dist/$1";
-		push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/apache/$1";
-		push @mirrors, "https://mirrors.ustc.edu.cn/apache/$1";
+		projectsmirrors '@APACHE', $1;
 	} elsif ($mirror =~ /^\@GITHUB\/(.+)$/) {
 		# give github a few more tries (different mirrors)
 		for (1 .. 5) {
-			push @mirrors, "https://raw.githubusercontent.com/$1";
+			projectsmirrors '@GITHUB', $1;
 		}
 	} elsif ($mirror =~ /^\@GNU\/(.+)$/) {
-		push @mirrors, "https://ftpmirror.gnu.org/$1";
-		push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnu/$1";
-		push @mirrors, "https://mirror.netcologne.de/gnu/$1";
-		push @mirrors, "https://ftp.kddilabs.jp/GNU/gnu/$1";
-		push @mirrors, "https://www.nic.funet.fi/pub/gnu/gnu/$1";
-		push @mirrors, "https://mirror.navercorp.com/gnu/$1";
-		push @mirrors, "https://mirrors.rit.edu/gnu/$1";
-		push @mirrors, "https://ftp.gnu.org/gnu/$1";
-		push @mirrors, "https://mirrors.tuna.tsinghua.edu.cn/gnu/$1";
-		push @mirrors, "https://mirrors.ustc.edu.cn/gnu/$1";
+		projectsmirrors '@GNU', $1;
 	} elsif ($mirror =~ /^\@SAVANNAH\/(.+)$/) {
-		push @mirrors, "https://download.savannah.nongnu.org/releases/$1";
-		push @mirrors, "https://mirror.netcologne.de/savannah/$1";
-		push @mirrors, "https://mirror.csclub.uwaterloo.ca/nongnu/$1";
-		push @mirrors, "https://ftp.acc.umu.se/mirror/gnu.org/savannah/$1";
-		push @mirrors, "https://nongnu.uib.no/$1";
-		push @mirrors, "https://cdimage.debian.org/mirror/gnu.org/savannah/$1";
+		projectsmirrors '@SAVANNAH', $1;
 	} elsif ($mirror =~ /^\@KERNEL\/(.+)$/) {
 		my @extra = ( $1 );
 		if ($filename =~ /linux-\d+\.\d+(?:\.\d+)?-rc/) {
@@ -297,29 +284,16 @@ sub cleanup
 			push @extra, "$extra[0]/longterm/v$1";
 		}
 		foreach my $dir (@extra) {
-			push @mirrors, "https://cdn.kernel.org/pub/$dir";
-			push @mirrors, "https://mirrors.mit.edu/kernel/$dir";
-			push @mirrors, "http://ftp.nara.wide.ad.jp/pub/kernel.org/$dir";
-			push @mirrors, "http://www.ring.gr.jp/archives/linux/kernel.org/$dir";
-			push @mirrors, "https://ftp.riken.jp/Linux/kernel.org/$dir";
-			push @mirrors, "https://www.mirrorservice.org/sites/ftp.kernel.org/pub/$dir";
-			push @mirrors, "https://mirrors.ustc.edu.cn/kernel.org/$dir";
+			projectsmirrors '@KERNEL', $dir;
 		}
 	} elsif ($mirror =~ /^\@GNOME\/(.+)$/) {
-		push @mirrors, "https://download.gnome.org/sources/$1";
-		push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnome/sources/$1";
-		push @mirrors, "https://ftp.acc.umu.se/pub/GNOME/sources/$1";
-		push @mirrors, "http://ftp.cse.buffalo.edu/pub/Gnome/sources/$1";
-		push @mirrors, "http://ftp.nara.wide.ad.jp/pub/X11/GNOME/sources/$1";
-		push @mirrors, "https://mirrors.ustc.edu.cn/gnome/sources/$1";
+		projectsmirrors '@GNOME', $1;
 	} else {
 		push @mirrors, $mirror;
 	}
 }
 
-push @mirrors, 'https://sources.cdn.openwrt.org';
-push @mirrors, 'https://sources.openwrt.org';
-push @mirrors, 'https://mirror2.openwrt.org/sources';
+projectsmirrors '@OPENWRT';
 
 if (-f "$target/$filename") {
 	$hash_cmd and do {

diff --git a/scripts/projectsmirrors.json b/scripts/projectsmirrors.json
@@ -0,0 +1,70 @@
+{
+	"@SF": [
+		"https://downloads.sourceforge.net"
+	],
+	"@DEBIAN": [
+		"https://ftp.debian.org/debian",
+		"https://mirror.leaseweb.com/debian",
+		"https://mirror.netcologne.de/debian",
+		"https://mirrors.tuna.tsinghua.edu.cn/debian",
+		"https://mirrors.ustc.edu.cn/debian"
+	],
+	"@APACHE": [
+		"https://dlcdn.apache.org",
+		"https://mirror.aarnet.edu.au/pub/apache",
+		"https://mirror.csclub.uwaterloo.ca/apache",
+		"https://archive.apache.org/dist",
+		"https://mirror.cogentco.com/pub/apache",
+		"https://mirror.navercorp.com/apache",
+		"https://ftp.jaist.ac.jp/pub/apache",
+		"https://apache.cs.utah.edu/apache.org",
+		"http://apache.mirrors.ovh.net/ftp.apache.org/dist",
+		"https://mirrors.tuna.tsinghua.edu.cn/apache",
+		"https://mirrors.ustc.edu.cn/apache"
+	],
+        "@GITHUB": [
+                "https://raw.githubusercontent.com"
+        ],
+	"@GNU": [
+		"https://ftpmirror.gnu.org",
+		"https://mirror.csclub.uwaterloo.ca/gnu",
+		"https://mirror.netcologne.de/gnu",
+		"https://ftp.kddilabs.jp/GNU/gnu",
+		"https://www.nic.funet.fi/pub/gnu/gnu",
+		"https://mirror.navercorp.com/gnu",
+		"https://mirrors.rit.edu/gnu",
+		"https://ftp.gnu.org/gnu",
+		"https://mirrors.tuna.tsinghua.edu.cn/gnu",
+		"https://mirrors.ustc.edu.cn/gnu"
+	],
+	"@SAVANNAH": [
+		"https://download.savannah.nongnu.org/releases",
+		"https://mirror.netcologne.de/savannah",
+		"https://mirror.csclub.uwaterloo.ca/nongnu",
+		"https://ftp.acc.umu.se/mirror/gnu.org/savannah",
+		"https://nongnu.uib.no",
+		"https://cdimage.debian.org/mirror/gnu.org/savannah"
+	],
+	"@KERNEL": [
+		"https://cdn.kernel.org/pub",
+		"https://mirrors.mit.edu/kernel",
+		"http://ftp.nara.wide.ad.jp/pub/kernel.org",
+		"http://www.ring.gr.jp/archives/linux/kernel.org",
+		"https://ftp.riken.jp/Linux/kernel.org",
+		"https://www.mirrorservice.org/sites/ftp.kernel.org/pub",
+		"https://mirrors.ustc.edu.cn/kernel.org"
+	],
+	"@GNOME": [
+		"https://download.gnome.org/sources",
+		"https://mirror.csclub.uwaterloo.ca/gnome/sources",
+		"https://ftp.acc.umu.se/pub/GNOME/sources",
+		"http://ftp.cse.buffalo.edu/pub/Gnome/sources",
+		"http://ftp.nara.wide.ad.jp/pub/X11/GNOME/sources",
+		"https://mirrors.ustc.edu.cn/gnome/sources"
+	],
+	"@OPENWRT": [
+		"https://sources.cdn.openwrt.org",
+		"https://sources.openwrt.org",
+		"https://mirror2.openwrt.org/sources"
+	]
+}