Maintenance: refactor servers updater code

- Require at least 80% of number of servers now to pass
- Each provider is in its own package with a common structure
- Unzip package with unzipper interface
- Openvpn package with extraction and download functions

internal/updater/providers/hidemyass/hosts.go

@@ -0,0 +1,19 @@
+package hidemyass
+
+func getUniqueHosts(tcpHostToURL, udpHostToURL map[string]string) (
+	hosts []string) {
+	uniqueHosts := make(map[string]struct{}, len(tcpHostToURL))
+	for host := range tcpHostToURL {
+		uniqueHosts[host] = struct{}{}
+	}
+	for host := range udpHostToURL {
+		uniqueHosts[host] = struct{}{}
+	}
+
+	hosts = make([]string, 0, len(uniqueHosts))
+	for host := range uniqueHosts {
+		hosts = append(hosts, host)
+	}
+
+	return hosts
+}

internal/updater/providers/hidemyass/hosttourl.go

@@ -0,0 +1,37 @@
+package hidemyass
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/updater/openvpn"
+)
+
+func getAllHostToURL(ctx context.Context, client *http.Client) (
+	tcpHostToURL, udpHostToURL map[string]string, err error) {
+	tcpHostToURL, err = getHostToURL(ctx, client, "TCP")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	udpHostToURL, err = getHostToURL(ctx, client, "UDP")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return tcpHostToURL, udpHostToURL, nil
+}
+
+func getHostToURL(ctx context.Context, client *http.Client, protocol string) (
+	hostToURL map[string]string, err error) {
+	const baseURL = "https://vpn.hidemyass.com/vpn-config"
+	indexURL := baseURL + "/" + strings.ToUpper(protocol) + "/"
+
+	urls, err := fetchIndex(ctx, client, indexURL)
+	if err != nil {
+		return nil, err
+	}
+
+	return openvpn.FetchMultiFiles(ctx, client, urls)
+}

internal/updater/providers/hidemyass/index.go

@@ -0,0 +1,54 @@
+package hidemyass
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+var indexOpenvpnLinksRegex = regexp.MustCompile(`<a[ ]+href=".+\.ovpn">.+\.ovpn</a>`)
+
+func fetchIndex(ctx context.Context, client *http.Client, indexURL string) (
+	openvpnURLs []string, err error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	htmlCode, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if !strings.HasSuffix(indexURL, "/") {
+		indexURL += "/"
+	}
+
+	lines := strings.Split(string(htmlCode), "\n")
+	for _, line := range lines {
+		found := indexOpenvpnLinksRegex.FindString(line)
+		if len(found) == 0 {
+			continue
+		}
+		const prefix = `.ovpn">`
+		const suffix = `</a>`
+		startIndex := strings.Index(found, prefix) + len(prefix)
+		endIndex := strings.Index(found, suffix)
+		filename := found[startIndex:endIndex]
+		openvpnURL := indexURL + filename
+		if !strings.HasSuffix(openvpnURL, ".ovpn") {
+			continue
+		}
+		openvpnURLs = append(openvpnURLs, openvpnURL)
+	}
+
+	return openvpnURLs, nil
+}

internal/updater/providers/hidemyass/resolve.go

@@ -0,0 +1,33 @@
+package hidemyass
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+)
+
+func resolveHosts(ctx context.Context, presolver resolver.Parallel,
+	hosts []string, minServers int) (
+	hostToIPs map[string][]net.IP, warnings []string, err error) {
+	const (
+		maxFailRatio    = 0.1
+		maxDuration     = 15 * time.Second
+		betweenDuration = 2 * time.Second
+		maxNoNew        = 2
+		maxFails        = 2
+	)
+	settings := resolver.ParallelSettings{
+		MaxFailRatio: maxFailRatio,
+		MinFound:     minServers,
+		Repeat: resolver.RepeatSettings{
+			MaxDuration:     maxDuration,
+			BetweenDuration: betweenDuration,
+			MaxNoNew:        maxNoNew,
+			MaxFails:        maxFails,
+			SortIPs:         true,
+		},
+	}
+	return presolver.Resolve(ctx, hosts, settings)
+}

internal/updater/providers/hidemyass/servers.go

@@ -0,0 +1,68 @@
+// Package hidemyass contains code to obtain the server information
+// for the HideMyAss provider.
+package hidemyass
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+)
+
+var ErrNotEnoughServers = errors.New("not enough servers found")
+
+func GetServers(ctx context.Context, client *http.Client,
+	presolver resolver.Parallel, minServers int) (
+	servers []models.HideMyAssServer, warnings []string, err error) {
+	tcpHostToURL, udpHostToURL, err := getAllHostToURL(ctx, client)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	hosts := getUniqueHosts(tcpHostToURL, udpHostToURL)
+
+	if len(hosts) < minServers {
+		return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(hosts), minServers)
+	}
+
+	hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	servers = make([]models.HideMyAssServer, 0, len(hostToIPs))
+	for host, IPs := range hostToIPs {
+		tcpURL, tcp := tcpHostToURL[host]
+		udpURL, udp := udpHostToURL[host]
+
+		// These two are only used to extract the country, region and city.
+		var url, protocol string
+		if tcp {
+			url = tcpURL
+			protocol = "TCP"
+		} else if udp {
+			url = udpURL
+			protocol = "UDP"
+		}
+		country, region, city := parseOpenvpnURL(url, protocol)
+
+		server := models.HideMyAssServer{
+			Country:  country,
+			Region:   region,
+			City:     city,
+			Hostname: host,
+			IPs:      IPs,
+			TCP:      tcp,
+			UDP:      udp,
+		}
+		servers = append(servers, server)
+	}
+
+	sortServers(servers)
+
+	return servers, warnings, nil
+}

internal/updater/providers/hidemyass/sort.go

@@ -0,0 +1,22 @@
+package hidemyass
+
+import (
+	"sort"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func sortServers(servers []models.HideMyAssServer) {
+	sort.Slice(servers, func(i, j int) bool {
+		if servers[i].Country == servers[j].Country {
+			if servers[i].Region == servers[j].Region {
+				if servers[i].City == servers[j].City {
+					return servers[i].Hostname < servers[j].Hostname
+				}
+				return servers[i].City < servers[j].City
+			}
+			return servers[i].Region < servers[j].Region
+		}
+		return servers[i].Country < servers[j].Country
+	})
+}

internal/updater/providers/hidemyass/string.go

@@ -0,0 +1,14 @@
+package hidemyass
+
+import "github.com/qdm12/gluetun/internal/models"
+
+func Stringify(servers []models.HideMyAssServer) (s string) {
+	s = "func HideMyAssServers() []models.HideMyAssServer {\n"
+	s += "	return []models.HideMyAssServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}

internal/updater/providers/hidemyass/url.go

@@ -0,0 +1,44 @@
+package hidemyass
+
+import (
+	"strings"
+	"unicode"
+)
+
+func parseOpenvpnURL(url, protocol string) (country, region, city string) {
+	lastSlashIndex := strings.LastIndex(url, "/")
+	url = url[lastSlashIndex+1:]
+
+	suffix := "." + strings.ToUpper(protocol) + ".ovpn"
+	url = strings.TrimSuffix(url, suffix)
+
+	parts := strings.Split(url, ".")
+
+	switch len(parts) {
+	case 1:
+		country = parts[0]
+		return country, "", ""
+	case 2: //nolint:gomnd
+		country = parts[0]
+		city = parts[1]
+	default:
+		country = parts[0]
+		region = parts[1]
+		city = parts[2]
+	}
+
+	return camelCaseToWords(country), camelCaseToWords(region),
+		camelCaseToWords(city)
+}
+
+func camelCaseToWords(camelCase string) (words string) {
+	wasLowerCase := false
+	for _, r := range camelCase {
+		if wasLowerCase && unicode.IsUpper(r) {
+			words += " "
+		}
+		wasLowerCase = unicode.IsLower(r)
+		words += string(r)
+	}
+	return words
+}