New PIA servers support (#227)

* Adapt storage: SyncServers write to file option, export FlushToFile
* CLI built-in updater for old and new PIA servers
* Update hardcoded IP addresses for PIA old and new servers
* Add PIA old to allServers struct and update timestamps
* Adapt code to work with new and old PIA servers
* Remove PIA subdomains (unneeded) from resolver tool

internal/updater/openvpn.go

@@ -0,0 +1,28 @@
+package updater
+
+import (
+	"net"
+	"strings"
+)
+
+func extractRemoteLinesFromOpenvpn(content []byte) (remoteLines []string) {
+	lines := strings.Split(string(content), "\n")
+	for _, line := range lines {
+		if strings.HasPrefix(line, "remote ") {
+			remoteLines = append(remoteLines, line)
+		}
+	}
+	return remoteLines
+}
+
+func extractIPsFromRemoteLines(remoteLines []string) (ips []net.IP) {
+	for _, remoteLine := range remoteLines {
+		fields := strings.Fields(remoteLine)
+		ip := net.ParseIP(fields[1])
+		if ip == nil { // not an IP address
+			continue
+		}
+		ips = append(ips, ip)
+	}
+	return ips
+}

internal/updater/options.go

@@ -0,0 +1,8 @@
+package updater
+
+type Options struct {
+	PIA    bool
+	PIAold bool
+	File   bool // update JSON file (user side)
+	Stdout bool // update constants file (maintainer side)
+}

internal/updater/pia.go

@@ -0,0 +1,66 @@
+package updater
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+func findPIAServers(new bool) (servers []models.PIAServer, err error) {
+	zipURL := "https://www.privateinternetaccess.com/openvpn/openvpn-ip.zip"
+	if new {
+		zipURL = "https://www.privateinternetaccess.com/openvpn/openvpn-ip-nextgen.zip"
+	}
+	return findPIAServersFromURL(zipURL)
+}
+
+func findPIAServersFromURL(zipURL string) (servers []models.PIAServer, err error) {
+	contents, err := fetchAndExtractFiles(zipURL)
+	if err != nil {
+		return nil, err
+	}
+	for fileName, content := range contents {
+		remoteLines := extractRemoteLinesFromOpenvpn(content)
+		if len(remoteLines) == 0 {
+			return nil, fmt.Errorf("cannot find any remote lines in %s", fileName)
+		}
+		IPs := extractIPsFromRemoteLines(remoteLines)
+		if len(IPs) == 0 {
+			return nil, fmt.Errorf("cannot find any IP addresses in %s", fileName)
+		}
+		region := strings.TrimSuffix(fileName, ".ovpn")
+		server := models.PIAServer{
+			Region: region,
+			IPs:    IPs,
+		}
+		servers = append(servers, server)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i].Region < servers[j].Region
+	})
+	return servers, nil
+}
+
+func stringifyPIAServers(servers []models.PIAServer) (s string) {
+	s = "func PIAServers() []models.PIAServer {\n"
+	s += "	return []models.PIAServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}
+
+func stringifyPIAOldServers(servers []models.PIAServer) (s string) {
+	s = "func PIAOldServers() []models.PIAServer {\n"
+	s += "	return []models.PIAServer{\n"
+	for _, server := range servers {
+		s += "		" + server.String() + ",\n"
+	}
+	s += "	}\n"
+	s += "}"
+	return s
+}

internal/updater/updater.go

@@ -0,0 +1,69 @@
+package updater
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/constants"
+	"github.com/qdm12/gluetun/internal/storage"
+)
+
+type Updater interface {
+	UpdateServers(options Options) error
+}
+
+type updater struct {
+	storage storage.Storage
+	timeNow func() time.Time
+	println func(s string)
+}
+
+func New(storage storage.Storage) Updater {
+	return &updater{
+		storage: storage,
+		timeNow: time.Now,
+		println: func(s string) { fmt.Println(s) },
+	}
+}
+
+func (u *updater) UpdateServers(options Options) error {
+	const writeSync = false
+	allServers, err := u.storage.SyncServers(constants.GetAllServers(), writeSync)
+	if err != nil {
+		return fmt.Errorf("cannot update servers: %w", err)
+	}
+
+	if options.PIA {
+		const newServers = true
+		servers, err := findPIAServers(newServers)
+		if err != nil {
+			return fmt.Errorf("cannot update PIA servers: %w", err)
+		}
+		if options.Stdout {
+			u.println(stringifyPIAServers(servers))
+		}
+		allServers.Pia.Timestamp = u.timeNow().Unix()
+		allServers.Pia.Servers = servers
+	}
+
+	if options.PIAold {
+		const newServers = false
+		servers, err := findPIAServers(newServers)
+		if err != nil {
+			return fmt.Errorf("cannot update PIA old servers: %w", err)
+		}
+		if options.Stdout {
+			u.println(stringifyPIAOldServers(servers))
+		}
+		allServers.PiaOld.Timestamp = u.timeNow().Unix()
+		allServers.PiaOld.Servers = servers
+	}
+
+	if options.File {
+		if err := u.storage.FlushToFile(allServers); err != nil {
+			return fmt.Errorf("cannot update servers: %w", err)
+		}
+	}
+
+	return nil
+}

internal/updater/zip.go

@@ -0,0 +1,62 @@
+package updater
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/qdm12/golibs/network"
+)
+
+func fetchAndExtractFiles(urls ...string) (contents map[string][]byte, err error) {
+	client := network.NewClient(10 * time.Second)
+	contents = make(map[string][]byte)
+	for _, url := range urls {
+		zipBytes, status, err := client.GetContent(url)
+		if err != nil {
+			return nil, err
+		} else if status != http.StatusOK {
+			return nil, fmt.Errorf("Getting %s results in HTTP status code %d", url, status)
+		}
+		newContents, err := zipExtractAll(zipBytes)
+		if err != nil {
+			return nil, err
+		}
+		for fileName, content := range newContents {
+			contents[fileName] = content
+		}
+	}
+	return contents, nil
+}
+
+func zipExtractAll(zipBytes []byte) (contents map[string][]byte, err error) {
+	r, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+	if err != nil {
+		return nil, err
+	}
+	contents = map[string][]byte{}
+	for _, zf := range r.File {
+		fileName := filepath.Base(zf.Name)
+		if !strings.HasSuffix(fileName, ".ovpn") {
+			continue
+		}
+		f, err := zf.Open()
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		contents[fileName], err = ioutil.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+		if err := f.Close(); err != nil {
+			return nil, err
+		}
+	}
+	return contents, nil
+}