feat: SlickVPN Support (#961)

- `internal/updater/html` package
- Add unit tests for slickvpn updating code
- Change shared html package to be more share-able
- Split html utilities in multiple files
- Fix processing .ovpn files with prefix space

Authored by @Rohaq 
Co-authored-by: Quentin McGaw <quentin.mcgaw@gmail.com>

.github/ISSUE_TEMPLATE/bug.yml

@@ -54,6 +54,7 @@ body:
         - PrivateVPN
         - ProtonVPN
         - PureVPN
+        - SlickVPN
         - Surfshark
         - TorGuard
         - VPNUnlimited

.github/labels.yml

@@ -64,6 +64,9 @@
 - name: ":cloud: PureVPN"
   color: "cfe8d4"
   description: ""
+- name: ":cloud: SlickVPN"
+  color: "cfe8d4"
+  description: ""
 - name: ":cloud: Surfshark"
   color: "cfe8d4"
   description: ""

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/qdm12/updated v0.0.0-20210603204757-205acfe6937e
 	github.com/stretchr/testify v1.8.0
 	github.com/vishvananda/netlink v1.1.1-0.20211129163951-9ada19101fc5
+	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 	golang.org/x/text v0.3.7
 	golang.zx2c4.com/wireguard v0.0.0-20210805125648-3957e9b9dd19
@@ -40,6 +41,5 @@ require (
 	go4.org/intern v0.0.0-20210108033219-3eb7198706b2 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
-	golang.org/x/net v0.0.0-20210504132125-bbd867fde50d // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -179,8 +179,9 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210504132125-bbd867fde50d h1:nTDGCTeAu2LhcsHTRzjyIUbZHCJ4QePArsm27Hka0UM=
 golang.org/x/net v0.0.0-20210504132125-bbd867fde50d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=

internal/configuration/settings/openvpnselection.go

@@ -82,6 +82,9 @@ func (o OpenVPNSelection) validate(vpnProvider string) (err error) {
 			case providers.Protonvpn:
 				allowedTCP = []uint16{443, 5995, 8443}
 				allowedUDP = []uint16{80, 443, 1194, 4569, 5060}
+			case providers.SlickVPN:
+				allowedTCP = []uint16{443, 8080, 8888}
+				allowedUDP = []uint16{443, 8080, 8888}
 			case providers.Wevpn:
 				allowedTCP = []uint16{53, 1195, 1199, 2018}
 				allowedUDP = []uint16{80, 1194, 1198}

internal/constants/providers/providers.go

@@ -19,6 +19,7 @@ const (
 	Privatevpn            = "privatevpn"
 	Protonvpn             = "protonvpn"
 	Purevpn               = "purevpn"
+	SlickVPN              = "slickvpn"
 	Surfshark             = "surfshark"
 	Torguard              = "torguard"
 	VPNUnlimited          = "vpn unlimited"
@@ -44,6 +45,7 @@ func All() []string {
 		Privatevpn,
 		Protonvpn,
 		Purevpn,
+		SlickVPN,
 		Surfshark,
 		Torguard,
 		VPNUnlimited,

internal/models/markdown.go

@@ -123,6 +123,8 @@ func getMarkdownHeaders(vpnProvider string) (headers []string) {
 		return []string{countryHeader, regionHeader, cityHeader, hostnameHeader, freeHeader}
 	case providers.Purevpn:
 		return []string{countryHeader, regionHeader, cityHeader, hostnameHeader, tcpHeader, udpHeader}
+	case providers.SlickVPN:
+		return []string{regionHeader, countryHeader, cityHeader, hostnameHeader}
 	case providers.Surfshark:
 		return []string{regionHeader, countryHeader, cityHeader, hostnameHeader, multiHopHeader, tcpHeader, udpHeader}
 	case providers.Torguard:

internal/provider/hidemyass/updater/hosttourl.go

@@ -33,5 +33,11 @@ func getHostToURL(ctx context.Context, client *http.Client, protocol string) (
 		return nil, err
 	}
 
-	return openvpn.FetchMultiFiles(ctx, client, urls)
+	const failEarly = true
+	hostToURL, errors := openvpn.FetchMultiFiles(ctx, client, urls, failEarly)
+	if len(errors) > 0 {
+		return nil, errors[0]
+	}
+
+	return hostToURL, nil
 }

internal/provider/providers.go

@@ -25,6 +25,7 @@ import (
 	"github.com/qdm12/gluetun/internal/provider/privatevpn"
 	"github.com/qdm12/gluetun/internal/provider/protonvpn"
 	"github.com/qdm12/gluetun/internal/provider/purevpn"
+	"github.com/qdm12/gluetun/internal/provider/slickvpn"
 	"github.com/qdm12/gluetun/internal/provider/surfshark"
 	"github.com/qdm12/gluetun/internal/provider/torguard"
 	"github.com/qdm12/gluetun/internal/provider/vpnunlimited"
@@ -71,6 +72,7 @@ func NewProviders(storage Storage, timeNow func() time.Time,
 		providers.Privatevpn:            privatevpn.New(storage, randSource, unzipper, updaterWarner, parallelResolver),
 		providers.Protonvpn:             protonvpn.New(storage, randSource, client, updaterWarner, parallelResolver),
 		providers.Purevpn:               purevpn.New(storage, randSource, ipFetcher, unzipper, updaterWarner, parallelResolver),
+		providers.SlickVPN:              slickvpn.New(storage, randSource, client, updaterWarner, parallelResolver),
 		providers.Surfshark:             surfshark.New(storage, randSource, client, unzipper, updaterWarner, parallelResolver),
 		providers.Torguard:              torguard.New(storage, randSource, unzipper, updaterWarner, parallelResolver),
 		providers.VPNUnlimited:          vpnunlimited.New(storage, randSource, unzipper, updaterWarner, parallelResolver),

internal/provider/slickvpn/connection.go

@@ -0,0 +1,13 @@
+package slickvpn
+
+import (
+	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/provider/utils"
+)
+
+func (p *Provider) GetConnection(selection settings.ServerSelection) (
+	connection models.Connection, err error) {
+	defaults := utils.NewConnectionDefaults(0, 443, 0) //nolint:gomnd
+	return utils.GetConnection(p.Name(), p.storage, selection, defaults, p.randSource)
+}

internal/provider/slickvpn/openvpnconf.go

@@ -0,0 +1,30 @@
+package slickvpn
+
+import (
+	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/constants/openvpn"
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/provider/utils"
+)
+
+func (p *Provider) OpenVPNConfig(connection models.Connection,
+	settings settings.OpenVPN) (lines []string) {
+	const pingSeconds = 10
+	const bufSize = 393216
+	providerSettings := utils.OpenVPNProviderSettings{
+		RemoteCertTLS: true,
+		AuthUserPass:  true,
+		Ciphers: []string{
+			openvpn.AES256cbc,
+		},
+		Ping:   pingSeconds,
+		SndBuf: bufSize,
+		RcvBuf: bufSize,
+		// Certificate found from https://www.slickvpn.com/tutorials/using-openvpn-configuration-files/
+		CA: "MIIESDCCAzCgAwIBAgIJAKHK5bbBPSU2MA0GCSqGSIb3DQEBBQUAMHUxCzAJBgNVBAYTAlVTMQwwCgYDVQQIEwNWUE4xDDAKBgNVBAcTA1ZQTjEMMAoGA1UEChMDVlBOMQwwCgYDVQQLEwNWUE4xDDAKBgNVBAMTA1ZQTjEMMAoGA1UEKRMDVlBOMRIwEAYJKoZIhvcNAQkBFgNWUE4wHhcNMjIwMjE0MjEzNDQwWhcNMzIwMjEyMjEzNDQwWjB1MQswCQYDVQQGEwJVUzEMMAoGA1UECBMDVlBOMQwwCgYDVQQHEwNWUE4xDDAKBgNVBAoTA1ZQTjEMMAoGA1UECxMDVlBOMQwwCgYDVQQDEwNWUE4xDDAKBgNVBCkTA1ZQTjESMBAGCSqGSIb3DQEJARYDVlBOMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwUl1XkfGo3c1uFsvgbO3C3yvu0+cHs9IUSsju5U9cPNCo53mqRHU/qntCC+ldIDKN+dNWn7eURIDszy+flutkgucs0qgETy5fzpXnVMtiKmMiOYWiJDor7j7QivRaxoT/O2fyjxvVCL8gMa60ekWSGBT6isYY8t8BnwTPVP0KvDm36wdaqLmubhf2XGvka/hhNx0SXMmz2x3OJ8BcoypZVLLk/+Qm6DJh1KxyDi4kI+jBC41QuaKKDNwb0kth1304eqZoUeCXtGkzl91y76ODAfdqzXf9WYJdgkXpOm53Cg7FtB42AqPRqHJVwYxDnQyrFwy4a3CWqFJnKtxJM/WlwIDAQABo4HaMIHXMB0GA1UdDgQWBBRSzxAtISfbSRPr0fmhwNZc8kOeKzCBpwYDVR0jBIGfMIGcgBRSzxAtISfbSRPr0fmhwNZc8kOeK6F5pHcwdTELMAkGA1UEBhMCVVMxDDAKBgNVBAgTA1ZQTjEMMAoGA1UEBxMDVlBOMQwwCgYDVQQKEwNWUE4xDDAKBgNVBAsTA1ZQTjEMMAoGA1UEAxMDVlBOMQwwCgYDVQQpEwNWUE4xEjAQBgkqhkiG9w0BCQEWA1ZQToIJAKHK5bbBPSU2MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAGuKFW765F3D5wax5IFSQbEtr+rVHgjR8jiYTzxOCmbLaU4oj6phOhfQJiTTADQYgCIC/DN0HsAEEqrKkwEn8KdAoNiAWfqCV/eqnK83y7yRDGx6/zfsch+PAzKZouMJLrvR9RYbHq7m3adZv84YLge7FE1JqFk1j6rSa4dUUnGQPrQgr9Sasnz8O8KK45XH6fqKrsd4p485n+BXPDzWVsHl4M5dqQV7qUZTazbzzh4NyP5/RQ6Oh5jqMN7po4qiqWv1pu0EKDxUG6gcECc2cTQwHhIOPeCGdHS7uuI2FlLnHaCUFBgi8zTsZxaeaPuPch5M7Zxbdg0GBhS2SmNi+io=", //nolint:lll
+		ExtraLines: []string{
+			"redirect-gateway",
+		},
+	}
+	return utils.OpenVPNConfig(providerSettings, connection, settings)
+}

internal/provider/slickvpn/provider.go

@@ -0,0 +1,33 @@
+package slickvpn
+
+import (
+	"math/rand"
+	"net/http"
+
+	"github.com/qdm12/gluetun/internal/constants/providers"
+	"github.com/qdm12/gluetun/internal/provider/common"
+	"github.com/qdm12/gluetun/internal/provider/slickvpn/updater"
+	"github.com/qdm12/gluetun/internal/provider/utils"
+)
+
+type Provider struct {
+	storage    common.Storage
+	randSource rand.Source
+	utils.NoPortForwarder
+	common.Fetcher
+}
+
+func New(storage common.Storage, randSource rand.Source,
+	client *http.Client, updaterWarner common.Warner,
+	parallelResolver common.ParallelResolver) *Provider {
+	return &Provider{
+		storage:         storage,
+		randSource:      randSource,
+		NoPortForwarder: utils.NewNoPortForwarding(providers.SlickVPN),
+		Fetcher:         updater.New(client, updaterWarner, parallelResolver),
+	}
+}
+
+func (p *Provider) Name() string {
+	return providers.SlickVPN
+}

internal/provider/slickvpn/updater/helpers_test.go

@@ -0,0 +1,26 @@
+package updater
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/html"
+)
+
+func parseTestHTML(t *testing.T, htmlString string) *html.Node {
+	t.Helper()
+	rootNode, err := html.Parse(strings.NewReader(htmlString))
+	require.NoError(t, err)
+	return rootNode
+}
+
+func parseTestDataIndexHTML(t *testing.T) *html.Node {
+	t.Helper()
+
+	data, err := os.ReadFile("testdata/index.html")
+	require.NoError(t, err)
+
+	return parseTestHTML(t, string(data))
+}

internal/provider/slickvpn/updater/resolve.go

@@ -0,0 +1,28 @@
+package updater
+
+import (
+	"time"
+
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+)
+
+func parallelResolverSettings(hosts []string) (settings resolver.ParallelSettings) {
+	const (
+		maxFailRatio    = 0.1
+		maxDuration     = 20 * time.Second
+		betweenDuration = time.Second
+		maxNoNew        = 2
+		maxFails        = 2
+	)
+	return resolver.ParallelSettings{
+		Hosts:        hosts,
+		MaxFailRatio: maxFailRatio,
+		Repeat: resolver.RepeatSettings{
+			MaxDuration:     maxDuration,
+			BetweenDuration: betweenDuration,
+			MaxNoNew:        maxNoNew,
+			MaxFails:        maxFails,
+			SortIPs:         true,
+		},
+	}
+}

internal/provider/slickvpn/updater/servers.go

@@ -0,0 +1,82 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/qdm12/gluetun/internal/constants/vpn"
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/provider/common"
+	"github.com/qdm12/gluetun/internal/updater/openvpn"
+)
+
+func (u *Updater) FetchServers(ctx context.Context, minServers int) (
+	servers []models.Server, err error) {
+	hostToData, err := fetchServers(ctx, u.client)
+	if err != nil {
+		return nil, fmt.Errorf("fetching and parsing website: %w", err)
+	}
+
+	openvpnURLs := make([]string, 0, len(hostToData))
+	for _, data := range hostToData {
+		openvpnURLs = append(openvpnURLs, data.ovpnURL)
+	}
+
+	if len(openvpnURLs) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(openvpnURLs), minServers)
+	}
+
+	const failEarly = false // some URLs from the website are not valid
+	udpHostToURL, errors := openvpn.FetchMultiFiles(ctx, u.client, openvpnURLs, failEarly)
+	for _, err := range errors {
+		u.warner.Warn(fmt.Sprintf("fetching OpenVPN files: %s", err))
+	}
+
+	if len(udpHostToURL) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(udpHostToURL), minServers)
+	}
+
+	hosts := make([]string, 0, len(udpHostToURL))
+	for host := range udpHostToURL {
+		hosts = append(hosts, host)
+	}
+
+	resolveSettings := parallelResolverSettings(hosts)
+	hostToIPs, warnings, err := u.parallelResolver.Resolve(ctx, resolveSettings)
+	for _, warning := range warnings {
+		u.warner.Warn(warning)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("resolving hosts: %w", err)
+	}
+
+	if len(hostToIPs) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			common.ErrNotEnoughServers, len(hosts), minServers)
+	}
+
+	servers = make([]models.Server, 0, len(hostToIPs))
+	for host, IPs := range hostToIPs {
+		_, udp := udpHostToURL[host]
+
+		serverData := hostToData[host]
+
+		server := models.Server{
+			VPN:      vpn.OpenVPN,
+			Region:   serverData.region,
+			Country:  serverData.country,
+			City:     serverData.city,
+			Hostname: host,
+			UDP:      udp,
+			IPs:      IPs,
+		}
+		servers = append(servers, server)
+	}
+
+	sort.Sort(models.SortableServers(servers))
+
+	return servers, nil
+}