Maintenance: refactor servers updater code

- Require at least 80% of number of servers now to pass
- Each provider is in its own package with a common structure
- Unzip package with unzipper interface
- Openvpn package with extraction and download functions

internal/updater/providers/mullvad/api.go

@@ -0,0 +1,55 @@
+package mullvad
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+var (
+	ErrHTTPStatusCodeNotOK   = errors.New("HTTP status code not OK")
+	ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
+)
+
+type serverData struct {
+	Hostname string `json:"hostname"`
+	Country  string `json:"country_name"`
+	City     string `json:"city_name"`
+	Active   bool   `json:"active"`
+	Owned    bool   `json:"owned"`
+	Provider string `json:"provider"`
+	IPv4     string `json:"ipv4_addr_in"`
+	IPv6     string `json:"ipv6_addr_in"`
+}
+
+func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) {
+	const url = "https://api.mullvad.net/www/relays/openvpn/"
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
+	}
+
+	decoder := json.NewDecoder(response.Body)
+	if err := decoder.Decode(&data); err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
+	}
+
+	if err := response.Body.Close(); err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}

internal/updater/providers/mullvad/hosttoserver.go

@@ -0,0 +1,58 @@
+package mullvad
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+type hostToServer map[string]models.MullvadServer
+
+var (
+	ErrParseIPv4 = errors.New("cannot parse IPv4 address")
+	ErrParseIPv6 = errors.New("cannot parse IPv6 address")
+)
+
+func (hts hostToServer) add(data serverData) (err error) {
+	if !data.Active {
+		return
+	}
+
+	ipv4 := net.ParseIP(data.IPv4)
+	if ipv4 == nil || ipv4.To4() == nil {
+		return fmt.Errorf("%w: %s", ErrParseIPv4, data.IPv4)
+	}
+
+	ipv6 := net.ParseIP(data.IPv6)
+	if ipv6 == nil || ipv6.To4() != nil {
+		return fmt.Errorf("%w: %s", ErrParseIPv6, data.IPv6)
+	}
+
+	server, ok := hts[data.Hostname]
+	if !ok {
+		server.Country = data.Country
+		server.City = strings.ReplaceAll(data.City, ",", "")
+		server.ISP = data.Provider
+		server.Owned = data.Owned
+	}
+
+	server.IPs = append(server.IPs, ipv4)
+	server.IPsV6 = append(server.IPsV6, ipv6)
+
+	hts[data.Hostname] = server
+
+	return nil
+}
+
+func (hts hostToServer) toServersSlice() (servers []models.MullvadServer) {
+	servers = make([]models.MullvadServer, 0, len(hts))
+	for _, server := range hts {
+		server.IPs = uniqueSortedIPs(server.IPs)
+		server.IPsV6 = uniqueSortedIPs(server.IPsV6)
+		servers = append(servers, server)
+	}
+	return servers
+}

internal/updater/providers/mullvad/ips.go

@@ -0,0 +1,30 @@
+package mullvad
+
+import (
+	"bytes"
+	"net"
+	"sort"
+)
+
+func uniqueSortedIPs(ips []net.IP) []net.IP {
+	uniqueIPs := make(map[string]struct{}, len(ips))
+	for _, ip := range ips {
+		key := ip.String()
+		uniqueIPs[key] = struct{}{}
+	}
+
+	ips = make([]net.IP, 0, len(uniqueIPs))
+	for key := range uniqueIPs {
+		ip := net.ParseIP(key)
+		if ipv4 := ip.To4(); ipv4 != nil {
+			ip = ipv4
+		}
+		ips = append(ips, ip)
+	}
+
+	sort.Slice(ips, func(i, j int) bool {
+		return bytes.Compare(ips[i], ips[j]) < 0
+	})
+
+	return ips
+}

internal/updater/providers/mullvad/ips_test.go

@@ -0,0 +1,41 @@
+package mullvad
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_uniqueSortedIPs(t *testing.T) {
+	t.Parallel()
+	testCases := map[string]struct {
+		inputIPs  []net.IP
+		outputIPs []net.IP
+	}{
+		"nil": {
+			inputIPs:  nil,
+			outputIPs: []net.IP{},
+		},
+		"empty": {
+			inputIPs:  []net.IP{},
+			outputIPs: []net.IP{},
+		},
+		"single IPv4": {
+			inputIPs:  []net.IP{{1, 1, 1, 1}},
+			outputIPs: []net.IP{{1, 1, 1, 1}},
+		},
+		"two IPv4s": {
+			inputIPs:  []net.IP{{1, 1, 2, 1}, {1, 1, 1, 1}},
+			outputIPs: []net.IP{{1, 1, 1, 1}, {1, 1, 2, 1}},
+		},
+	}
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			outputIPs := uniqueSortedIPs(testCase.inputIPs)
+			assert.Equal(t, testCase.outputIPs, outputIPs)
+		})
+	}
+}

internal/updater/providers/mullvad/servers.go

@@ -0,0 +1,68 @@
+// Package mullvad contains code to obtain the server information
+// for the Mullvad provider.
+package mullvad
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+var ErrNotEnoughServers = errors.New("not enough servers found")
+
+func GetServers(ctx context.Context, client *http.Client, minServers int) (
+	servers []models.MullvadServer, err error) {
+	data, err := fetchAPI(ctx, client)
+	if err != nil {
+		return nil, err
+	}
+
+	hts := make(hostToServer)
+	for _, serverData := range data {
+		if err := hts.add(serverData); err != nil {
+			return nil, err
+		}
+	}
+
+	if len(hts) < minServers {
+		return nil, fmt.Errorf("%w: %d and expected at least %d",
+			ErrNotEnoughServers, len(hts), minServers)
+	}
+
+	servers = hts.toServersSlice()
+
+	servers = groupByProperties(servers)
+
+	sortServers(servers)
+
+	return servers, nil
+}
+
+// TODO group by hostname so remove this.
+func groupByProperties(serversByHost []models.MullvadServer) (serversByProps []models.MullvadServer) {
+	propsToServer := make(map[string]models.MullvadServer, len(serversByHost))
+	for _, server := range serversByHost {
+		key := server.Country + server.City + server.ISP + strconv.FormatBool(server.Owned)
+		serverByProps, ok := propsToServer[key]
+		if !ok {
+			serverByProps.Country = server.Country
+			serverByProps.City = server.City
+			serverByProps.ISP = server.ISP
+			serverByProps.Owned = server.Owned
+		}
+		serverByProps.IPs = append(serverByProps.IPs, server.IPs...)
+		serverByProps.IPsV6 = append(serverByProps.IPsV6, server.IPsV6...)
+		propsToServer[key] = serverByProps
+	}
+
+	serversByProps = make([]models.MullvadServer, 0, len(propsToServer))
+	for _, serverByProp := range propsToServer {
+		serversByProps = append(serversByProps, serverByProp)
+	}
+
+	return serversByProps
+}

internal/updater/providers/mullvad/sort.go

@@ -0,0 +1,19 @@
+package mullvad
+
+import (
+	"sort"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func sortServers(servers []models.MullvadServer) {
+	sort.Slice(servers, func(i, j int) bool {
+		if servers[i].Country == servers[j].Country {
+			if servers[i].City == servers[j].City {
+				return servers[i].ISP < servers[j].ISP
+			}
+			return servers[i].City < servers[j].City
+		}
+		return servers[i].Country < servers[j].Country
+	})
+}

internal/updater/providers/mullvad/string.go

@@ -0,0 +1,14 @@
+package mullvad
+
+import "github.com/qdm12/gluetun/internal/models"
+
+func Stringify(servers []models.MullvadServer) (s string) {
+	s = "func MullvadServers() []models.MullvadServer {\n"
+	s += "	return []models.MullvadServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}

internal/updater/providers/mullvad/string_test.go

@@ -0,0 +1,32 @@
+package mullvad
+
+import (
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/qdm12/gluetun/internal/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Stringify(t *testing.T) {
+	servers := []models.MullvadServer{{
+		Country: "webland",
+		City:    "webcity",
+		ISP:     "not nsa",
+		Owned:   true,
+		IPs:     []net.IP{{1, 1, 1, 1}},
+		IPsV6:   []net.IP{{1, 1, 1, 1}},
+	}}
+	//nolint:lll
+	expected := `
+func MullvadServers() []models.MullvadServer {
+	return []models.MullvadServer{
+		{Country: "webland", City: "webcity", ISP: "not nsa", Owned: true, IPs: []net.IP{{1, 1, 1, 1}}, IPsV6: []net.IP{{1, 1, 1, 1}}},
+	}
+}
+`
+	expected = strings.TrimPrefix(strings.TrimSuffix(expected, "\n"), "\n")
+	s := Stringify(servers)
+	assert.Equal(t, expected, s)
+}