Skip to content

Commit

Permalink
chore: refactor mirrors manage
Browse files Browse the repository at this point in the history
  • Loading branch information
@soulteary
soulteary committed Nov 19, 2022
1 parent d8e742d commit 1715caa
Show file tree
Hide file tree
Showing 9 changed files with 220 additions and 65 deletions.
19 changes: 19 additions & 0 deletions internal/define/alpine.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package define

import "regexp"

var ALPINE_HOST_PATTERN = regexp.MustCompile(`https?://.+/alpine/(.+)$`)

const ALPINE_BENCHMAKR_URL = "MIRRORS.txt"

// https://mirrors.alpinelinux.org/ 2022.11.19
var BUILDIN_ALPINE_MIRRORS = []UrlWithAlias{
{URL: "mirrors.tuna.tsinghua.edu.cn/alpine/", Alias: "cn:tsinghua", Http: true, Https: true, Official: true, Bandwidth: 10000},
{URL: "mirrors.ustc.edu.cn/alpine/", Alias: "cn:ustc", Http: true, Https: true, Official: true, Bandwidth: 700},
{URL: "mirrors.nju.edu.cn/alpine/", Alias: "cn:nju", Http: true, Https: true, Official: true, Bandwidth: 10000},
{URL: "mirror.lzu.edu.cn/alpine/", Alias: "cn:lzu", Http: true, Https: true, Official: true, Bandwidth: 100},
{URL: "mirrors.sjtug.sjtu.edu.cn/alpine/", Alias: "cn:sjtug", Http: true, Https: true, Official: true, Bandwidth: 500},
{URL: "mirrors.aliyun.com/alpine/", Alias: "cn:aliyun", Http: true, Https: true, Official: true, Bandwidth: 10000},
{URL: "mirrors.bfsu.edu.cn/centos/", Alias: "cn:bfsu", Http: true, Https: true, Official: true, Bandwidth: 1000},
{URL: "mirrors.neusoft.edu.cn/centos/", Alias: "cn:neusoft", Http: true, Https: true, Official: true, Bandwidth: 1000},
}
28 changes: 15 additions & 13 deletions internal/define/centos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,20 @@ var CENTOS_HOST_PATTERN = regexp.MustCompile(`https?://.+/centos/(.+)$`)

const CENTOS_BENCHMAKR_URL = "TIME"

// https://www.centos.org/download/mirrors/
var BUILDIN_CENTOS_MIRRORS = []UrlWithAlias{
{URL: "https://mirrors.tuna.tsinghua.edu.cn/centos/", Alias: "cn:tsinghua"},
{URL: "http://mirrors.aliyun.com/centos/", Alias: "cn:aliyun"},
{URL: "https://mirrors.bfsu.edu.cn/centos/", Alias: "cn:bfsu"},
{URL: "https://mirrors.cqu.edu.cn/CentOS/", Alias: "cn:cqu"},
{URL: "http://mirrors.neusoft.edu.cn/centos/", Alias: "cn:neusoft"},
{URL: "https://mirror.nju.edu.cn/centos/", Alias: "cn:nju"},
{URL: "https://mirrors.huaweicloud.com/centos/", Alias: "cn:huaweicloud"},
{URL: "http://mirror.lzu.edu.cn/centos/", Alias: "cn:lzu"},
{URL: "https://mirrors.njupt.edu.cn/centos/", Alias: "cn:njupt"},
{URL: "http://mirrors.163.com/centos/", Alias: "cn:163"},
{URL: "https://mirrors.bupt.edu.cn/centos/", Alias: "cn:bupt"},
{URL: "https://ftp.sjtu.edu.cn/centos/", Alias: "cn:sjtu"},
{URL: "https://mirrors.ustc.edu.cn/centos/", Alias: "cn:ustc"},
{URL: "https://mirrors.tuna.tsinghua.edu.cn/centos/", Alias: "cn:tsinghua", Http: true, Https: true, Official: true},
{URL: "http://mirrors.aliyun.com/centos/", Alias: "cn:aliyun", Http: true, Https: true, Official: true},
{URL: "https://mirrors.bfsu.edu.cn/centos/", Alias: "cn:bfsu", Http: true, Https: true, Official: true},
{URL: "https://mirrors.cqu.edu.cn/CentOS/", Alias: "cn:cqu", Http: true, Https: true, Official: true},
{URL: "https://mirror.nju.edu.cn/centos/", Alias: "cn:nju", Http: true, Https: true, Official: true},
{URL: "http://mirror.lzu.edu.cn/centos/", Alias: "cn:lzu", Http: true, Https: true, Official: true},
{URL: "https://mirrors.njupt.edu.cn/centos/", Alias: "cn:njupt", Http: true, Https: true, Official: true},
{URL: "http://mirrors.163.com/centos/", Alias: "cn:163", Http: true, Https: true, Official: true},
{URL: "https://mirrors.bupt.edu.cn/centos/", Alias: "cn:bupt", Http: true, Https: true, Official: true},
{URL: "https://ftp.sjtu.edu.cn/centos/", Alias: "cn:sjtu", Http: true, Https: true, Official: true},
{URL: "https://mirrors.ustc.edu.cn/centos/", Alias: "cn:ustc", Http: true, Https: true, Official: true},
// TODO: valid?
// {URL: "http://mirrors.neusoft.edu.cn/centos/", Alias: "cn:neusoft"},
{URL: "https://mirrors.huaweicloud.com/centos/", Alias: "cn:huaweicloud", Http: true, Https: true, Official: false},
}
25 changes: 13 additions & 12 deletions internal/define/debian.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,19 @@ var DEBIAN_HOST_PATTERN = regexp.MustCompile(
`https?://(deb|security|snapshot).debian.org/debian/(.+)$`,
)

// https://www.debian.org/mirror/list 2022.11.19
var BUILDIN_DEBIAN_MIRRORS = []UrlWithAlias{
{URL: "https://mirrors.tuna.tsinghua.edu.cn/debian/", Alias: "cn:tsinghua"},
{URL: "http://mirrors.ustc.edu.cn/debian/", Alias: "cn:ustc"},
{URL: "https://mirrors.163.com/debian/", Alias: "cn:163"},
{URL: "https://mirrors.aliyun.com/debian/", Alias: "cn:aliyun"},
{URL: "https://repo.huaweicloud.com/debian/", Alias: "cn:huawei"},
{URL: "https://mirrors.tuna.tsinghua.edu.cn/debian/", Alias: "cn:tsinghua", Http: true, Https: true, Official: true},
{URL: "http://mirrors.ustc.edu.cn/debian/", Alias: "cn:ustc", Http: true, Https: true, Official: true},
{URL: "https://mirrors.163.com/debian/", Alias: "cn:163", Http: true, Https: true, Official: true},
{URL: "https://repo.huaweicloud.com/debian/", Alias: "cn:huawei", Http: true, Https: true, Official: true},
{URL: "https://mirrors.cloud.tencent.com/debian/", Alias: "cn:tencent"},
{URL: "http://ftp.cn.debian.org/debian/", Alias: "cn:debian"},
{URL: "http://mirror.bjtu.edu.cn/debian/", Alias: "cn:bjtu"},
{URL: "http://mirror.lzu.edu.cn/debian/", Alias: "cn:lzu"},
{URL: "http://mirror.nju.edu.cn/debian/", Alias: "cn:nju"},
{URL: "http://mirrors.bfsu.edu.cn/debian/", Alias: "cn:bfsu"},
{URL: "http://mirrors.hit.edu.cn/debian/", Alias: "cn:hit"},
{URL: "http://mirrors.neusoft.edu.cn/debian/", Alias: "cn:neusoft"},
{URL: "http://ftp.cn.debian.org/debian/", Alias: "cn:debian", Http: true, Https: true, Official: true},
{URL: "http://mirror.bjtu.edu.cn/debian/", Alias: "cn:bjtu", Http: true, Https: true, Official: true},
{URL: "http://mirrors.bfsu.edu.cn/debian/", Alias: "cn:bfsu", Http: true, Https: true, Official: true},
{URL: "http://mirrors.neusoft.edu.cn/debian/", Alias: "cn:neusoft", Http: true, Https: true, Official: true},
{URL: "http://mirrors.hit.edu.cn/debian/", Alias: "cn:hit", Http: true, Https: true, Official: false},
{URL: "https://mirrors.aliyun.com/debian/", Alias: "cn:aliyun", Http: true, Https: true, Official: false},
{URL: "http://mirror.lzu.edu.cn/debian/", Alias: "cn:lzu", Http: true, Https: true, Official: false},
{URL: "http://mirror.nju.edu.cn/debian/", Alias: "cn:nju", Http: true, Https: true, Official: false},
}
62 changes: 60 additions & 2 deletions internal/define/define.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package define
import (
"fmt"
"regexp"
"strings"
)

const (
Expand Down Expand Up @@ -32,6 +33,63 @@ func (r *Rule) String() string {
}

type UrlWithAlias struct {
URL string
Alias string
URL string
Alias string
Http bool
Https bool
Official bool
Bandwidth int64
}

func GenerateAliasFromURL(url string) string {
pureHost := regexp.MustCompile(`^https?://|\/.*`).ReplaceAllString(url, "")
tldRemoved := regexp.MustCompile(`\.edu\.cn$|.cn$|\.com$|\.net$|\.net.cn$|\.org$|\.org\.cn$`).ReplaceAllString(pureHost, "")
group := strings.Split(tldRemoved, ".")
return "cn:" + group[len(group)-1]
}

func GenerateBuildInMirorItem(url string, offical bool) UrlWithAlias {
var mirror UrlWithAlias
mirror.Official = offical
mirror.Alias = GenerateAliasFromURL(url)

if strings.HasPrefix(url, "http://") {
mirror.Http = true
mirror.Https = false
} else if strings.HasPrefix(url, "https://") {
mirror.Http = false
mirror.Https = true
}
mirror.URL = url
// TODO
mirror.Bandwidth = 0
return mirror
}

func GenerateBuildInList(officialList []string, customList []string) (mirrors []UrlWithAlias) {
for _, url := range officialList {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
mirror := GenerateBuildInMirorItem("http://"+url, true)
mirrors = append(mirrors, mirror)
mirror = GenerateBuildInMirorItem("https://"+url, true)
mirrors = append(mirrors, mirror)
} else {
mirror := GenerateBuildInMirorItem(url, true)
mirrors = append(mirrors, mirror)
}
}

for _, url := range customList {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
mirror := GenerateBuildInMirorItem("http://"+url, false)
mirrors = append(mirrors, mirror)
mirror = GenerateBuildInMirorItem("https://"+url, false)
mirrors = append(mirrors, mirror)
} else {
mirror := GenerateBuildInMirorItem(url, false)
mirrors = append(mirrors, mirror)
}
}

return mirrors
}
60 changes: 60 additions & 0 deletions internal/define/define_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,22 @@ package define_test
import (
"fmt"
"regexp"
"strings"
"testing"

Define "github.com/soulteary/apt-proxy/internal/define"
)

// func TestPrintUbuntuPingScript(t *testing.T) {
// for _, url := range UBUNTU_OFFICAL_MIRRORS {
// fmt.Println(`echo "` + url + `"`)
// http := "curl --connect-timeout 2 -I http://" + url + UBUNTU_BENCHMAKR_URL
// fmt.Println(http)
// https := "curl --connect-timeout 2 -I https://" + url + UBUNTU_BENCHMAKR_URL
// fmt.Println(https)
// }
// }

func TestRuleToString(t *testing.T) {
r := Define.Rule{
Pattern: regexp.MustCompile(`a$`),
Expand All @@ -20,3 +31,52 @@ func TestRuleToString(t *testing.T) {
t.Fatal("parse rule to string failed")
}
}

func TestGenerateAliasFromURL(t *testing.T) {
if Define.GenerateAliasFromURL("http://mirrors.cn99.com/ubuntu/") != "cn:cn99" {
t.Fatal("generate alias from url failed")
}

if Define.GenerateAliasFromURL("https://mirrors.tuna.tsinghua.edu.cn/ubuntu/") != "cn:tsinghua" {
t.Fatal("generate alias from url failed")
}

if Define.GenerateAliasFromURL("mirrors.cnnic.cn/ubuntu/") != "cn:cnnic" {
t.Fatal("generate alias from url failed")
}
}

func TestGenerateBuildInMirorItem(t *testing.T) {
mirror := Define.GenerateBuildInMirorItem("http://mirrors.tuna.tsinghua.edu.cn/ubuntu/", true)
if !(mirror.Http == true && mirror.Https == false) || mirror.Official != true {
t.Fatal("generate build-in mirror item failed")
}
mirror = Define.GenerateBuildInMirorItem("https://mirrors.tuna.tsinghua.edu.cn/ubuntu/", false)
if !(mirror.Http == false && mirror.Https == true) || mirror.Official != false {
t.Fatal("generate build-in mirror item failed")
}
}

func TestGenerateBuildInList(t *testing.T) {
mirrors := Define.GenerateBuildInList(Define.UBUNTU_OFFICAL_MIRRORS, Define.UBUNTU_CUSTOM_MIRRORS)

count := 0
for _, url := range Define.UBUNTU_OFFICAL_MIRRORS {
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
count += 1
} else {
count += 2
}
}
for _, url := range Define.UBUNTU_CUSTOM_MIRRORS {
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
count += 1
} else {
count += 2
}
}

if len(mirrors) != count {
t.Fatal("generate build-in mirror list failed")
}
}
62 changes: 38 additions & 24 deletions internal/define/ubuntu.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package define

import "regexp"
import (
"regexp"
)

const (
UBUNTU_GEO_MIRROR_API = "http://mirrors.ubuntu.com/mirrors.txt"
Expand All @@ -11,27 +13,39 @@ var UBUNTU_HOST_PATTERN = regexp.MustCompile(
`https?://(\w{2}\.)?(security|archive).ubuntu.com/ubuntu/(.+)$`,
)

var BUILDIN_UBUNTU_MIRRORS = []UrlWithAlias{
{URL: "https://mirrors.tuna.tsinghua.edu.cn/ubuntu/", Alias: "cn:tsinghua"},
{URL: "http://mirrors.ustc.edu.cn/ubuntu/", Alias: "cn:ustc"},
{URL: "https://mirrors.163.com/ubuntu/", Alias: "cn:163"},
{URL: "https://mirrors.aliyun.com/ubuntu/", Alias: "cn:aliyun"},
// "http://mirrors.huaweicloud.com/repository/ubuntu/"
{URL: "https://repo.huaweicloud.com/ubuntu/", Alias: "cn:huawei"},
{URL: "https://mirrors.cloud.tencent.com/ubuntu/", Alias: "cn:tencent"},
{URL: "http://mirror.dlut.edu.cn/ubuntu/", Alias: "cn:dlut"},
{URL: "http://mirrors.dgut.edu.cn/ubuntu/", Alias: "cn:dgut"},
{URL: "http://mirrors.njupt.edu.cn/ubuntu/", Alias: "cn:njupt"},
{URL: "https://mirrors.hit.edu.cn/ubuntu/", Alias: "cn:hit"},
{URL: "http://mirrors.yun-idc.com/ubuntu/", Alias: "cn:yun-idc"},
{URL: "http://ftp.sjtu.edu.cn/ubuntu/", Alias: "cn:sjtu"},
{URL: "https://mirror.nju.edu.cn/ubuntu/", Alias: "cn:nju"},
{URL: "https://mirrors.bupt.edu.cn/ubuntu/", Alias: "cn:bupt"},
{URL: "http://mirrors.skyshe.cn/ubuntu/", Alias: "cn:skyshe"},
{URL: "http://mirror.lzu.edu.cn/ubuntu/", Alias: "cn:lzu"},
{URL: "http://mirrors.cn99.com/ubuntu/", Alias: "cn:cn99"},
{URL: "http://mirrors.cqu.edu.cn/ubuntu/", Alias: "cn:cqu"},
{URL: "https://mirror.bjtu.edu.cn/ubuntu/", Alias: "cn:bjtu"},
{URL: "http://mirrors.sohu.com/ubuntu/", Alias: "cn:sohu"},
{URL: "http://cn.archive.ubuntu.com/ubuntu/", Alias: "cn:ubuntu"},
// http://mirrors.ubuntu.com/mirrors.txt 2022.11.19
// Sites that contain protocol headers, restrict access to resources using that protocol
var UBUNTU_OFFICAL_MIRRORS = []string{
"mirrors.cn99.com/ubuntu/",
"mirrors.tuna.tsinghua.edu.cn/ubuntu/",
"mirrors.cnnic.cn/ubuntu/",
"mirror.bjtu.edu.cn/ubuntu/",
"mirrors.cqu.edu.cn/ubuntu/",
"http://mirrors.skyshe.cn/ubuntu/",
// duplicate "mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/",
"mirrors.yun-idc.com/ubuntu/",
"http://mirror.dlut.edu.cn/ubuntu/",
"mirrors.xjtu.edu.cn/ubuntu/",
"mirrors.huaweicloud.com/repository/ubuntu/",
"mirrors.bupt.edu.cn/ubuntu/",
"mirrors.hit.edu.cn/ubuntu/",
// duplicate "repo.huaweicloud.com/ubuntu/",
"http://mirrors.sohu.com/ubuntu/",
"mirror.nju.edu.cn/ubuntu/",
"mirrors.bfsu.edu.cn/ubuntu/",
"mirror.lzu.edu.cn/ubuntu/",
"mirrors.aliyun.com/ubuntu/",
"ftp.sjtu.edu.cn/ubuntu/",
"mirrors.njupt.edu.cn/ubuntu/",
"mirrors.cloud.tencent.com/ubuntu/",
"http://mirrors.dgut.edu.cn/ubuntu/",
"mirrors.ustc.edu.cn/ubuntu/",
"mirrors.sdu.edu.cn/ubuntu/",
"http://cn.archive.ubuntu.com/ubuntu/",
}

var UBUNTU_CUSTOM_MIRRORS = []string{
"mirrors.163.com/ubuntu/",
}

var BUILDIN_UBUNTU_MIRRORS = GenerateBuildInList(UBUNTU_OFFICAL_MIRRORS, UBUNTU_CUSTOM_MIRRORS)
9 changes: 5 additions & 4 deletions internal/mirrors/custom_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package mirrors

import (
"strings"
"testing"

Define "github.com/soulteary/apt-proxy/internal/define"
)

func TestGetUbuntuMirrorByAliases(t *testing.T) {
alias := GetMirrorURLByAliases(Define.TYPE_LINUX_DISTROS_UBUNTU, "cn:tsinghua")
if alias != Define.BUILDIN_UBUNTU_MIRRORS[0].URL {
if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/ubuntu/") {
t.Fatal("Test Get Mirror By Custom Name Failed")
}

Expand All @@ -20,7 +21,7 @@ func TestGetUbuntuMirrorByAliases(t *testing.T) {

func TestGetDebianMirrorByAliases(t *testing.T) {
alias := GetMirrorURLByAliases(Define.TYPE_LINUX_DISTROS_DEBIAN, "cn:tsinghua")
if alias != Define.BUILDIN_DEBIAN_MIRRORS[0].URL {
if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/debian/") {
t.Fatal("Test Get Mirror By Custom Name Failed")
}

Expand All @@ -32,8 +33,8 @@ func TestGetDebianMirrorByAliases(t *testing.T) {

func TestGetCentOSMirrorByAliases(t *testing.T) {
alias := GetMirrorURLByAliases(Define.TYPE_LINUX_DISTROS_CENTOS, "cn:tsinghua")
if alias != Define.BUILDIN_CENTOS_MIRRORS[0].URL {
t.Fatal("Test Get Mirror By Custom Name Failed", alias)
if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/centos/") {
t.Fatal("Test Get Mirror By Custom Name Failed")
}

alias = GetMirrorURLByAliases(Define.TYPE_LINUX_DISTROS_CENTOS, "cn:not-found")
Expand Down
18 changes: 9 additions & 9 deletions internal/mirrors/mirrors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ func GenerateMirrorListByPredefined(osType int) (mirrors []string) {
return mirrors
}

var BUILDIN_OFFICAL_UBUNTU_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_UBUNTU)
var BUILDIN_OFFICAL_CENTOS_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_CENTOS)
var BUILDIN_OFFICAL_DEBIAN_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_DEBIAN)
var BUILDIN_UBUNTU_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_UBUNTU)
var BUILDIN_CENTOS_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_CENTOS)
var BUILDIN_DEBIAN_MIRRORS = GenerateMirrorListByPredefined(Define.TYPE_LINUX_DISTROS_DEBIAN)

var DEBIAN_DEFAULT_CACHE_RULES = []Define.Rule{
{Pattern: regexp.MustCompile(`deb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: Define.TYPE_LINUX_DISTROS_DEBIAN},
Expand Down Expand Up @@ -88,22 +88,22 @@ func GetGeoMirrorUrlsByMode(mode int) (mirrors []string) {
if mode == Define.TYPE_LINUX_DISTROS_UBUNTU {
ubuntuMirrorsOnline, err := getUbuntuMirrorUrlsByGeo()
if err != nil {
return BUILDIN_OFFICAL_UBUNTU_MIRRORS
return BUILDIN_UBUNTU_MIRRORS
}
return ubuntuMirrorsOnline
}

if mode == Define.TYPE_LINUX_DISTROS_DEBIAN {
return BUILDIN_OFFICAL_DEBIAN_MIRRORS
return BUILDIN_DEBIAN_MIRRORS
}

if mode == Define.TYPE_LINUX_DISTROS_CENTOS {
return BUILDIN_OFFICAL_CENTOS_MIRRORS
return BUILDIN_CENTOS_MIRRORS
}

mirrors = append(mirrors, BUILDIN_OFFICAL_UBUNTU_MIRRORS...)
mirrors = append(mirrors, BUILDIN_OFFICAL_DEBIAN_MIRRORS...)
mirrors = append(mirrors, BUILDIN_OFFICAL_CENTOS_MIRRORS...)
mirrors = append(mirrors, BUILDIN_UBUNTU_MIRRORS...)
mirrors = append(mirrors, BUILDIN_DEBIAN_MIRRORS...)
mirrors = append(mirrors, BUILDIN_CENTOS_MIRRORS...)
return mirrors
}

Expand Down
2 changes: 1 addition & 1 deletion internal/mirrors/mirrors_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestGetMirrorUrlsByGeo(t *testing.T) {
}

mirrors = GetGeoMirrorUrlsByMode(Define.TYPE_LINUX_DISTROS_DEBIAN)
if len(mirrors) != len(BUILDIN_OFFICAL_DEBIAN_MIRRORS) {
if len(mirrors) != len(BUILDIN_DEBIAN_MIRRORS) {
t.Fatal("Get mirrors error")
}

Expand Down

0 comments on commit 1715caa

Please sign in to comment.