Maintenance: refactor servers updater code

- Require at least 80% of number of servers now to pass
- Each provider is in its own package with a common structure
- Unzip package with unzipper interface
- Openvpn package with extraction and download functions

internal/updater/providers/surfshark/api.go

@@ -0,0 +1,53 @@
+package surfshark
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+var (
+	ErrHTTPStatusCodeNotOK   = errors.New("HTTP status code not OK")
+	ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
+)
+
+//nolint:unused
+type serverData struct {
+	Host     string `json:"connectionName"`
+	Country  string `json:"country"`
+	Location string `json:"location"`
+}
+
+//nolint:unused,deadcode
+func fetchAPI(ctx context.Context, client *http.Client) (
+	servers []serverData, err error) {
+	const url = "https://my.surfshark.com/vpn/api/v4/server/clusters"
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
+	}
+
+	decoder := json.NewDecoder(response.Body)
+	if err := decoder.Decode(&servers); err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
+	}
+
+	if err := response.Body.Close(); err != nil {
+		return nil, err
+	}
+
+	return servers, nil
+}

internal/updater/providers/surfshark/hosttoserver.go

@@ -0,0 +1,59 @@
+package surfshark
+
+import (
+	"net"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+type hostToServer map[string]models.SurfsharkServer
+
+func (hts hostToServer) add(host, region string) {
+	// TODO set TCP and UDP
+	// TODO set hostname
+	server, ok := hts[host]
+	if !ok {
+		server.Region = region
+		hts[host] = server
+	}
+}
+
+func (hts hostToServer) toHostsSlice() (hosts []string) {
+	hosts = make([]string, 0, len(hts))
+	for host := range hts {
+		hosts = append(hosts, host)
+	}
+	return hosts
+}
+
+func (hts hostToServer) toSubdomainsSlice() (subdomains []string) {
+	subdomains = make([]string, 0, len(hts))
+	const suffix = ".prod.surfshark.com"
+	for host := range hts {
+		subdomain := strings.TrimSuffix(host, suffix)
+		subdomains = append(subdomains, subdomain)
+	}
+	return subdomains
+}
+
+func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
+	for host, IPs := range hostToIPs {
+		server := hts[host]
+		server.IPs = IPs
+		hts[host] = server
+	}
+	for host, server := range hts {
+		if len(server.IPs) == 0 {
+			delete(hts, host)
+		}
+	}
+}
+
+func (hts hostToServer) toServersSlice() (servers []models.SurfsharkServer) {
+	servers = make([]models.SurfsharkServer, 0, len(hts))
+	for _, server := range hts {
+		servers = append(servers, server)
+	}
+	return servers
+}

internal/updater/providers/surfshark/regions.go

@@ -0,0 +1,200 @@
+package surfshark
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+var (
+	errSuffixNotFound    = errors.New("suffix not found")
+	errSubdomainNotFound = errors.New("subdomain not found in subdomain to region mapping")
+)
+
+func parseHost(host string, subdomainToRegion map[string]string) (
+	region string, err error) {
+	const suffix = ".prod.surfshark.com"
+	if !strings.HasSuffix(host, suffix) {
+		return "", fmt.Errorf("%w: %s in %s",
+			errSuffixNotFound, suffix, host)
+	}
+
+	subdomain := strings.TrimSuffix(host, suffix)
+	region, ok := subdomainToRegion[subdomain]
+	if !ok {
+		return "", fmt.Errorf("%w: %s", errSubdomainNotFound, subdomain)
+	}
+
+	return region, nil
+}
+
+func subdomainToRegion() (mapping map[string]string) {
+	return map[string]string{
+		"ae-dub":       "United Arab Emirates",
+		"al-tia":       "Albania",
+		"at-vie":       "Austria",
+		"au-adl":       "Australia Adelaide",
+		"au-bne":       "Australia Brisbane",
+		"au-mel":       "Australia Melbourne",
+		"au-per":       "Australia Perth",
+		"au-syd":       "Australia Sydney",
+		"au-us":        "Australia US",
+		"az-bak":       "Azerbaijan",
+		"ba-sjj":       "Bosnia and Herzegovina",
+		"be-bru":       "Belgium",
+		"bg-sof":       "Bulgaria",
+		"br-sao":       "Brazil",
+		"ca-mon":       "Canada Montreal",
+		"ca-tor":       "Canada Toronto",
+		"ca-us":        "Canada US",
+		"ca-van":       "Canada Vancouver",
+		"ch-zur":       "Switzerland",
+		"cl-san":       "Chile",
+		"co-bog":       "Colombia",
+		"cr-sjn":       "Costa Rica",
+		"cy-nic":       "Cyprus",
+		"cz-prg":       "Czech Republic",
+		"de-ber":       "Germany Berlin",
+		"de-fra":       "Germany Frankfurt am Main",
+		"de-fra-st001": "Germany Frankfurt am Main st001",
+		"de-fra-st002": "Germany Frankfurt am Main st002",
+		"de-fra-st003": "Germany Frankfurt am Main st003",
+		"de-fra-st004": "Germany Frankfurt am Main st004",
+		"de-fra-st005": "Germany Frankfurt am Main st005",
+		"de-muc":       "Germany Munich",
+		"de-nue":       "Germany Nuremberg",
+		"de-sg":        "Germany Singapour",
+		"de-uk":        "Germany UK",
+		"dk-cph":       "Denmark",
+		"ee-tll":       "Estonia",
+		"es-bcn":       "Spain Barcelona",
+		"es-mad":       "Spain Madrid",
+		"es-vlc":       "Spain Valencia",
+		"fi-hel":       "Finland",
+		"fr-bod":       "France Bordeaux",
+		"fr-mrs":       "France Marseilles",
+		"fr-par":       "France Paris",
+		"fr-se":        "France Sweden",
+		"gr-ath":       "Greece",
+		"hk-hkg":       "Hong Kong",
+		"hr-zag":       "Croatia",
+		"hu-bud":       "Hungary",
+		"id-jak":       "Indonesia",
+		"ie-dub":       "Ireland",
+		"il-tlv":       "Israel",
+		"in-chn":       "India Chennai",
+		"in-idr":       "India Indore",
+		"in-mum":       "India Mumbai",
+		"in-uk":        "India UK",
+		"is-rkv":       "Iceland",
+		"it-mil":       "Italy Milan",
+		"it-rom":       "Italy Rome",
+		"jp-tok":       "Japan Tokyo",
+		"jp-tok-st001": "Japan Tokyo st001",
+		"jp-tok-st002": "Japan Tokyo st002",
+		"jp-tok-st003": "Japan Tokyo st003",
+		"jp-tok-st004": "Japan Tokyo st004",
+		"jp-tok-st005": "Japan Tokyo st005",
+		"jp-tok-st006": "Japan Tokyo st006",
+		"jp-tok-st007": "Japan Tokyo st007",
+		"jp-tok-st008": "Japan Tokyo st008",
+		"jp-tok-st009": "Japan Tokyo st009",
+		"jp-tok-st010": "Japan Tokyo st010",
+		"jp-tok-st011": "Japan Tokyo st011",
+		"jp-tok-st012": "Japan Tokyo st012",
+		"jp-tok-st013": "Japan Tokyo st013",
+		"kr-seo":       "Korea",
+		"kz-ura":       "Kazakhstan",
+		"lu-ste":       "Luxembourg",
+		"lv-rig":       "Latvia",
+		"ly-tip":       "Libya",
+		"md-chi":       "Moldova",
+		"mk-skp":       "North Macedonia",
+		"my-kul":       "Malaysia",
+		"ng-lag":       "Nigeria",
+		"nl-ams":       "Netherlands Amsterdam",
+		"nl-ams-st001": "Netherlands Amsterdam st001",
+		"nl-us":        "Netherlands US",
+		"no-osl":       "Norway",
+		"nz-akl":       "New Zealand",
+		"ph-mnl":       "Philippines",
+		"pl-gdn":       "Poland Gdansk",
+		"pl-waw":       "Poland Warsaw",
+		"pt-lis":       "Portugal Lisbon",
+		"pt-lou":       "Portugal Loule",
+		"pt-opo":       "Portugal Porto",
+		"py-asu":       "Paraguay",
+		"ro-buc":       "Romania",
+		"rs-beg":       "Serbia",
+		"ru-mos":       "Russia Moscow",
+		"ru-spt":       "Russia St. Petersburg",
+		"se-sto":       "Sweden",
+		"sg-hk":        "Singapore Hong Kong",
+		"sg-nl":        "Singapore Netherlands",
+		"sg-sng":       "Singapore",
+		"sg-in":        "Singapore in",
+		"sg-sng-st001": "Singapore st001",
+		"sg-sng-st002": "Singapore st002",
+		"sg-sng-st003": "Singapore st003",
+		"sg-sng-st004": "Singapore st004",
+		"sg-sng-mp001": "Singapore mp001",
+		"si-lju":       "Slovenia",
+		"sk-bts":       "Slovekia",
+		"th-bkk":       "Thailand",
+		"tr-bur":       "Turkey",
+		"tw-tai":       "Taiwan",
+		"ua-iev":       "Ukraine",
+		"uk-de":        "UK Germany",
+		"uk-fr":        "UK France",
+		"uk-gla":       "UK Glasgow",
+		"uk-lon":       "UK London",
+		"uk-lon-mp001": "UK London mp001",
+		"uk-lon-st001": "UK London st001",
+		"uk-lon-st002": "UK London st002",
+		"uk-lon-st003": "UK London st003",
+		"uk-lon-st004": "UK London st004",
+		"uk-lon-st005": "UK London st005",
+		"uk-man":       "UK Manchester",
+		"us-atl":       "US Atlanta",
+		"us-bdn":       "US Bend",
+		"us-bos":       "US Boston",
+		"us-buf":       "US Buffalo",
+		"us-chi":       "US Chicago",
+		"us-clt":       "US Charlotte",
+		"us-dal":       "US Dallas",
+		"us-den":       "US Denver",
+		"us-dtw":       "US Gahanna",
+		"us-hou":       "US Houston",
+		"us-kan":       "US Kansas City",
+		"us-las":       "US Las Vegas",
+		"us-lax":       "US Los Angeles",
+		"us-ltm":       "US Latham",
+		"us-mia":       "US Miami",
+		"us-mnz":       "US Maryland",
+		"us-nl":        "US Netherlands",
+		"us-nyc":       "US New York City",
+		"us-nyc-mp001": "US New York City mp001",
+		"us-nyc-st001": "US New York City st001",
+		"us-nyc-st002": "US New York City st002",
+		"us-nyc-st003": "US New York City st003",
+		"us-nyc-st004": "US New York City st004",
+		"us-nyc-st005": "US New York City st005",
+		"us-orl":       "US Orlando",
+		"us-phx":       "US Phoenix",
+		"us-pt":        "US Portugal",
+		"us-sea":       "US Seatle",
+		"us-sfo":       "US San Francisco",
+		"us-slc":       "US Salt Lake City",
+		"us-stl":       "US Saint Louis",
+		"us-tpa":       "US Tampa",
+		"vn-hcm":       "Vietnam",
+		"za-jnb":       "South Africa",
+		"ar-bua":       "Argentina Buenos Aires",
+		"tr-ist":       "Turkey Istanbul",
+		"mx-mex":       "Mexico City Mexico",
+		"ca-tor-mp001": "Canada Toronto mp001",
+		"de-fra-mp001": "Germany Frankfurt mp001",
+		"nl-ams-mp001": "Netherlands Amsterdam mp001",
+		"us-sfo-mp001": "US San Francisco mp001",
+	}
+}

internal/updater/providers/surfshark/resolve.go

@@ -0,0 +1,32 @@
+package surfshark
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+)
+
+func resolveHosts(ctx context.Context, presolver resolver.Parallel,
+	hosts []string, minServers int) (hostToIPs map[string][]net.IP,
+	warnings []string, err error) {
+	const (
+		maxFailRatio    = 0.1
+		maxDuration     = 20 * time.Second
+		betweenDuration = time.Second
+		maxNoNew        = 2
+		maxFails        = 2
+	)
+	settings := resolver.ParallelSettings{
+		MaxFailRatio: maxFailRatio,
+		MinFound:     minServers,
+		Repeat: resolver.RepeatSettings{
+			MaxDuration:     maxDuration,
+			BetweenDuration: betweenDuration,
+			MaxNoNew:        maxNoNew,
+			MaxFails:        maxFails,
+		},
+	}
+	return presolver.Resolve(ctx, hosts, settings)
+}

internal/updater/providers/surfshark/servers.go

@@ -0,0 +1,130 @@
+// Package surfshark contains code to obtain the server information
+// for the Surshark provider.
+package surfshark
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/updater/openvpn"
+	"github.com/qdm12/gluetun/internal/updater/resolver"
+	"github.com/qdm12/gluetun/internal/updater/unzip"
+)
+
+var ErrNotEnoughServers = errors.New("not enough servers found")
+
+func GetServers(ctx context.Context, unzipper unzip.Unzipper,
+	presolver resolver.Parallel, minServers int) (
+	servers []models.SurfsharkServer, warnings []string, err error) {
+	const url = "https://my.surfshark.com/vpn/api/v1/server/configurations"
+	contents, err := unzipper.FetchAndExtract(ctx, url)
+	if err != nil {
+		return nil, nil, err
+	} else if len(contents) < minServers {
+		return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(contents), minServers)
+	}
+
+	subdomainToRegion := subdomainToRegion()
+	hts := make(hostToServer)
+
+	for fileName, content := range contents {
+		if !strings.HasSuffix(fileName, ".ovpn") {
+			continue // not an OpenVPN file
+		}
+
+		host, warning, err := openvpn.ExtractHost(content)
+		if warning != "" {
+			warnings = append(warnings, warning)
+		}
+		if err != nil {
+			// treat error as warning and go to next file
+			warning := err.Error() + " in " + fileName
+			warnings = append(warnings, warning)
+			continue
+		}
+
+		region, err := parseHost(host, subdomainToRegion)
+		if err != nil {
+			// treat error as warning and go to next file
+			warning := err.Error()
+			warnings = append(warnings, warning)
+			continue
+		}
+
+		hts.add(host, region)
+	}
+
+	if len(hts) < minServers {
+		return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(hts), minServers)
+	}
+
+	hosts := hts.toHostsSlice()
+	hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
+	warnings = append(warnings, newWarnings...)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	hts.adaptWithIPs(hostToIPs)
+
+	servers = hts.toServersSlice()
+
+	// process subdomain entries in mapping that were not in the Zip file
+	subdomainsDone := hts.toSubdomainsSlice()
+	for _, subdomainDone := range subdomainsDone {
+		delete(subdomainToRegion, subdomainDone)
+	}
+	remainingServers, newWarnings, err := getRemainingServers(
+		ctx, subdomainToRegion, presolver)
+	warnings = append(warnings, newWarnings...)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	servers = append(servers, remainingServers...)
+
+	if len(servers) < minServers {
+		return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(servers), minServers)
+	}
+
+	sortServers(servers)
+
+	return servers, warnings, nil
+}
+
+func getRemainingServers(ctx context.Context,
+	subdomainToRegionLeft map[string]string, presolver resolver.Parallel) (
+	servers []models.SurfsharkServer, warnings []string, err error) {
+	hosts := make([]string, 0, len(subdomainToRegionLeft))
+	const suffix = ".prod.surfshark.com"
+	for subdomain := range subdomainToRegionLeft {
+		hosts = append(hosts, subdomain+suffix)
+	}
+
+	const minServers = 0
+	hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
+	if err != nil {
+		return nil, warnings, err
+	}
+
+	servers = make([]models.SurfsharkServer, 0, len(hostToIPs))
+	for host, IPs := range hostToIPs {
+		region, err := parseHost(host, subdomainToRegionLeft)
+		if err != nil {
+			return nil, warnings, err
+		}
+		server := models.SurfsharkServer{
+			Region: region,
+			IPs:    IPs,
+		}
+		servers = append(servers, server)
+	}
+
+	return servers, warnings, nil
+}

internal/updater/providers/surfshark/sort.go

@@ -0,0 +1,13 @@
+package surfshark
+
+import (
+	"sort"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func sortServers(servers []models.SurfsharkServer) {
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i].Region < servers[j].Region
+	})
+}

internal/updater/providers/surfshark/string.go

@@ -0,0 +1,14 @@
+package surfshark
+
+import "github.com/qdm12/gluetun/internal/models"
+
+func Stringify(servers []models.SurfsharkServer) (s string) {
+	s = "func SurfsharkServers() []models.SurfsharkServer {\n"
+	s += "	return []models.SurfsharkServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}