admin/gluetun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Maintenance: refactor servers updater code

Browse Source 
- Require at least 80% of number of servers now to pass
- Each provider is in its own package with a common structure
- Unzip package with unzipper interface
- Openvpn package with extraction and download functions
This commit is contained in:
Quentin McGaw
2021-05-08 00:59:42 +00:00
parent 442340dcf2
commit e8e7b83297
107 changed files with 3778 additions and 2374 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/updater/errors.go
View File

@@ -1,9 +0,0 @@
package updater
import "errors"
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("cannot unmarshal response body")
ErrUpdateServerInformation = errors.New("failed updating server information")
)

161
internal/updater/fastestvpn.go
View File

@@ -1,161 +0,0 @@
package updater
import (
"context"
"fmt"
"net/http"
"regexp"
"sort"
"strings"
"time"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updateFastestvpn(ctx context.Context) (err error) {
servers, warnings, err := findFastestvpnServersFromZip(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("FastestVPN: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update FastestVPN servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyFastestVPNServers(servers))
}
u.servers.Fastestvpn.Timestamp = u.timeNow().Unix()
u.servers.Fastestvpn.Servers = servers
return nil
}
func findFastestvpnServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.FastestvpnServer, warnings []string, err error) {
const zipURL = "https://support.fastestvpn.com/download/openvpn-tcp-udp-config-files"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
trailNumberExp := regexp.MustCompile(`[0-9]+$`)
type Data struct {
TCP bool
UDP bool
Country string
}
hostToData := make(map[string]Data)
for fileName, content := range contents {
const (
tcpSuffix = "-TCP.ovpn"
udpSuffix = "-UDP.ovpn"
)
var tcp, udp bool
var suffix string
switch {
case strings.HasSuffix(fileName, tcpSuffix):
suffix = tcpSuffix
tcp = true
case strings.HasSuffix(fileName, udpSuffix):
suffix = udpSuffix
udp = true
default:
warning := `filename "` + fileName + `" does not have a protocol suffix`
warnings = append(warnings, warning)
continue
}
countryWithNumber := strings.TrimSuffix(fileName, suffix)
number := trailNumberExp.FindString(countryWithNumber)
country := countryWithNumber[:len(countryWithNumber)-len(number)]
host, warning, err := extractHostFromOVPN(content)
if len(warning) > 0 {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warnings = append(warnings, err.Error()+" in "+fileName)
continue
}
data := hostToData[host]
data.Country = country
if tcp {
data.TCP = true
}
if udp {
data.UDP = true
}
hostToData[host] = data
}
hosts := make([]string, len(hostToData))
i := 0
for host := range hostToData {
hosts[i] = host
i++
}
const (
maxFailRatio = 0.1
maxNoNew = 1
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: time.Second,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
for host, IPs := range hostToIPs {
if len(IPs) == 0 {
warning := fmt.Sprintf("no IP address found for host %q", host)
warnings = append(warnings, warning)
continue
}
data := hostToData[host]
server := models.FastestvpnServer{
Hostname: host,
TCP: data.TCP,
UDP: data.UDP,
Country: data.Country,
IPs: IPs,
}
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].Country < servers[j].Country
})
return servers, warnings, nil
}
func stringifyFastestVPNServers(servers []models.FastestvpnServer) (s string) {
s = "func FastestvpnServers() []models.FastestvpnServer {\n"
s += " return []models.FastestvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

288
internal/updater/hma.go
View File

@@ -1,288 +0,0 @@
package updater
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"sort"
"strings"
"time"
"unicode"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updateHideMyAss(ctx context.Context) (err error) {
servers, warnings, err := findHideMyAssServers(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("HideMyAss: %s", warning)
}
}
if err != nil {
return fmt.Errorf("%w: HideMyAss: %s", ErrUpdateServerInformation, err)
}
if u.options.Stdout {
u.println(stringifyHideMyAssServers(servers))
}
u.servers.HideMyAss.Timestamp = u.timeNow().Unix()
u.servers.HideMyAss.Servers = servers
return nil
}
func findHideMyAssServers(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.HideMyAssServer, warnings []string, err error) {
TCPhostToURL, err := findHideMyAssHostToURLForProto(ctx, client, "TCP")
if err != nil {
return nil, nil, err
}
UDPhostToURL, err := findHideMyAssHostToURLForProto(ctx, client, "UDP")
if err != nil {
return nil, nil, err
}
uniqueHosts := make(map[string]struct{}, len(TCPhostToURL))
for host := range TCPhostToURL {
uniqueHosts[host] = struct{}{}
}
for host := range UDPhostToURL {
uniqueHosts[host] = struct{}{}
}
hosts := make([]string, len(uniqueHosts))
i := 0
for host := range uniqueHosts {
hosts[i] = host
i++
}
const (
maxFailRatio = 0.1
maxDuration = 15 * time.Second
betweenDuration = 2 * time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings)
if err != nil {
return nil, warnings, err
}
servers = make([]models.HideMyAssServer, 0, len(hostToIPs))
for host, IPs := range hostToIPs {
tcpURL, tcp := TCPhostToURL[host]
udpURL, udp := UDPhostToURL[host]
var url, protocol string
if tcp {
url = tcpURL
protocol = "TCP"
} else if udp {
url = udpURL
protocol = "UDP"
}
country, region, city := parseHideMyAssURL(url, protocol)
server := models.HideMyAssServer{
Country: country,
Region: region,
City: city,
Hostname: host,
IPs: IPs,
TCP: tcp,
UDP: udp,
}
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].Country+servers[i].Region+servers[i].City+servers[i].Hostname <
servers[j].Country+servers[j].Region+servers[j].City+servers[j].Hostname
})
return servers, warnings, nil
}
func findHideMyAssHostToURLForProto(ctx context.Context, client *http.Client, protocol string) (
hostToURL map[string]string, err error) {
indexURL := "https://vpn.hidemyass.com/vpn-config/" + strings.ToUpper(protocol) + "/"
urls, err := fetchHideMyAssHTTPIndex(ctx, client, indexURL)
if err != nil {
return nil, err
}
return fetchMultiOvpnFiles(ctx, client, urls)
}
func parseHideMyAssURL(url, protocol string) (country, region, city string) {
lastSlashIndex := strings.LastIndex(url, "/")
url = url[lastSlashIndex+1:]
suffix := "." + strings.ToUpper(protocol) + ".ovpn"
url = strings.TrimSuffix(url, suffix)
parts := strings.Split(url, ".")
switch len(parts) {
case 1:
country = parts[0]
return country, "", ""
case 2: //nolint:gomnd
country = parts[0]
city = parts[1]
default:
country = parts[0]
region = parts[1]
city = parts[2]
}
return camelCaseToWords(country), camelCaseToWords(region), camelCaseToWords(city)
}
func camelCaseToWords(camelCase string) (words string) {
wasLowerCase := false
for _, r := range camelCase {
if wasLowerCase && unicode.IsUpper(r) {
words += " "
}
wasLowerCase = unicode.IsLower(r)
words += string(r)
}
return words
}
var hideMyAssIndexRegex = regexp.MustCompile(`<a[ ]+href=".+\.ovpn">.+\.ovpn</a>`)
func fetchHideMyAssHTTPIndex(ctx context.Context, client *http.Client, indexURL string) (urls []string, err error) {
htmlCode, err := fetchFile(ctx, client, indexURL)
if err != nil {
return nil, err
}
if !strings.HasSuffix(indexURL, "/") {
indexURL += "/"
}
lines := strings.Split(string(htmlCode), "\n")
for _, line := range lines {
found := hideMyAssIndexRegex.FindString(line)
if len(found) == 0 {
continue
}
const prefix = `.ovpn">`
const suffix = `</a>`
startIndex := strings.Index(found, prefix) + len(prefix)
endIndex := strings.Index(found, suffix)
filename := found[startIndex:endIndex]
url := indexURL + filename
if !strings.HasSuffix(url, ".ovpn") {
continue
}
urls = append(urls, url)
}
return urls, nil
}
func fetchMultiOvpnFiles(ctx context.Context, client *http.Client, urls []string) (
hostToURL map[string]string, err error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
hostToURL = make(map[string]string, len(urls))
type Result struct {
url string
host string
}
results := make(chan Result)
errors := make(chan error)
for _, url := range urls {
go func(url string) {
host, err := fetchOvpnFile(ctx, client, url)
if err != nil {
errors <- fmt.Errorf("%w: for %s", err, url)
return
}
results <- Result{
url: url,
host: host,
}
}(url)
}
for range urls {
select {
case newErr := <-errors:
if err == nil { // only assign to the first error
err = newErr
cancel() // stop other operations, this will trigger other errors we ignore
}
case result := <-results:
hostToURL[result.host] = result.url
}
}
if err != nil {
return nil, err
}
return hostToURL, nil
}
func fetchOvpnFile(ctx context.Context, client *http.Client, url string) (hostname string, err error) {
b, err := fetchFile(ctx, client, url)
if err != nil {
return "", err
}
const rejectIP = true
const rejectDomain = false
hosts := extractRemoteHostsFromOpenvpn(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", errRemoteHostNotFound
}
return hosts[0], nil
}
func fetchFile(ctx context.Context, client *http.Client, url string) (b []byte, err error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
return io.ReadAll(response.Body)
}
func stringifyHideMyAssServers(servers []models.HideMyAssServer) (s string) {
s = "func HideMyAssServers() []models.HideMyAssServer {\n"
s += " return []models.HideMyAssServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

112
internal/updater/mullvad.go
View File

@@ -1,112 +0,0 @@
package updater
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"sort"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
func (u *updater) updateMullvad(ctx context.Context) (err error) {
servers, err := findMullvadServers(ctx, u.client)
if err != nil {
return fmt.Errorf("cannot update Mullvad servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyMullvadServers(servers))
}
u.servers.Mullvad.Timestamp = u.timeNow().Unix()
u.servers.Mullvad.Servers = servers
return nil
}
func findMullvadServers(ctx context.Context, client *http.Client) (servers []models.MullvadServer, err error) {
const url = "https://api.mullvad.net/www/relays/openvpn/"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url)
}
decoder := json.NewDecoder(response.Body)
var data []struct {
Country string `json:"country_name"`
City string `json:"city_name"`
Active bool `json:"active"`
Owned bool `json:"owned"`
Provider string `json:"provider"`
IPv4 string `json:"ipv4_addr_in"`
IPv6 string `json:"ipv6_addr_in"`
}
if err := decoder.Decode(&data); err != nil {
return nil, err
}
if err := response.Body.Close(); err != nil {
return nil, err
}
serversByKey := map[string]models.MullvadServer{}
for _, jsonServer := range data {
if !jsonServer.Active {
continue
}
ipv4 := net.ParseIP(jsonServer.IPv4)
ipv6 := net.ParseIP(jsonServer.IPv6)
if ipv4 == nil || ipv4.To4() == nil {
return nil, fmt.Errorf("cannot parse ipv4 address %q", jsonServer.IPv4)
} else if ipv6 == nil || ipv6.To4() != nil {
return nil, fmt.Errorf("cannot parse ipv6 address %q", jsonServer.IPv6)
}
key := fmt.Sprintf("%s%s%t%s", jsonServer.Country, jsonServer.City, jsonServer.Owned, jsonServer.Provider)
if server, ok := serversByKey[key]; ok {
server.IPs = append(server.IPs, ipv4)
server.IPsV6 = append(server.IPsV6, ipv6)
serversByKey[key] = server
} else {
serversByKey[key] = models.MullvadServer{
IPs: []net.IP{ipv4},
IPsV6: []net.IP{ipv6},
Country: jsonServer.Country,
City: strings.ReplaceAll(jsonServer.City, ",", ""),
ISP: jsonServer.Provider,
Owned: jsonServer.Owned,
}
}
}
for _, server := range serversByKey {
server.IPs = uniqueSortedIPs(server.IPs)
server.IPsV6 = uniqueSortedIPs(server.IPsV6)
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].Country+servers[i].City+servers[i].ISP < servers[j].Country+servers[j].City+servers[j].ISP
})
return servers, nil
}
func stringifyMullvadServers(servers []models.MullvadServer) (s string) {
s = "func MullvadServers() []models.MullvadServer {\n"
s += " return []models.MullvadServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

125
internal/updater/nordvpn.go
View File

@@ -1,125 +0,0 @@
package updater
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"sort"
"strconv"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
func (u *updater) updateNordvpn(ctx context.Context) (err error) {
servers, warnings, err := findNordvpnServers(ctx, u.client)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Nordvpn: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Nordvpn servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyNordvpnServers(servers))
}
u.servers.Nordvpn.Timestamp = u.timeNow().Unix()
u.servers.Nordvpn.Servers = servers
return nil
}
var (
ErrNoIDInServerName = errors.New("no ID in server name")
ErrInvalidIDInServerName = errors.New("invalid ID in server name")
)
func findNordvpnServers(ctx context.Context, client *http.Client) (
servers []models.NordvpnServer, warnings []string, err error) {
const url = "https://nordvpn.com/api/server"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url)
}
decoder := json.NewDecoder(response.Body)
var data []struct {
IPAddress string `json:"ip_address"`
Name string `json:"name"`
Country string `json:"country"`
Features struct {
UDP bool `json:"openvpn_udp"`
TCP bool `json:"openvpn_tcp"`
} `json:"features"`
}
if err := decoder.Decode(&data); err != nil {
return nil, nil, err
}
if err := response.Body.Close(); err != nil {
return nil, nil, err
}
sort.Slice(data, func(i, j int) bool {
if data[i].Country == data[j].Country {
return data[i].Name < data[j].Name
}
return data[i].Country < data[j].Country
})
for _, jsonServer := range data {
if !jsonServer.Features.TCP && !jsonServer.Features.UDP {
warnings = append(warnings, fmt.Sprintf("server %q does not support TCP and UDP for openvpn", jsonServer.Name))
continue
}
ip := net.ParseIP(jsonServer.IPAddress)
if ip == nil || ip.To4() == nil {
return nil, nil,
fmt.Errorf("IP address %q is not a valid IPv4 address for server %q",
jsonServer.IPAddress, jsonServer.Name)
}
i := strings.IndexRune(jsonServer.Name, '#')
if i < 0 {
return nil, nil, fmt.Errorf("%w: %s", ErrNoIDInServerName, jsonServer.Name)
}
idString := jsonServer.Name[i+1:]
idUint64, err := strconv.ParseUint(idString, 10, 16)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidIDInServerName, jsonServer.Name)
}
server := models.NordvpnServer{
Region: jsonServer.Country,
Number: uint16(idUint64),
IP: ip,
TCP: jsonServer.Features.TCP,
UDP: jsonServer.Features.UDP,
}
servers = append(servers, server)
}
return servers, warnings, nil
}
func stringifyNordvpnServers(servers []models.NordvpnServer) (s string) {
s = "func NordvpnServers() []models.NordvpnServer {\n"
s += " return []models.NordvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

50
internal/updater/openvpn.go
View File

@@ -1,50 +0,0 @@
package updater
import (
"errors"
"fmt"
"net"
"strings"
)
var (
errRemoteHostNotFound = errors.New("remote host not found")
)
func extractHostFromOVPN(b []byte) (host, warning string, err error) {
const (
rejectIP = true
rejectDomain = false
)
hosts := extractRemoteHostsFromOpenvpn(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", "", errRemoteHostNotFound
} else if len(hosts) > 1 {
warning = fmt.Sprintf(
"only using the first host %q and discarding %d other hosts",
hosts[0], len(hosts)-1)
}
return hosts[0], warning, nil
}
func extractRemoteHostsFromOpenvpn(content []byte,
rejectIP, rejectDomain bool) (hosts []string) {
lines := strings.Split(string(content), "\n")
for _, line := range lines {
if !strings.HasPrefix(line, "remote ") {
continue
}
fields := strings.Fields(line)
if len(fields) == 1 || len(fields[1]) == 0 {
continue
}
host := fields[1]
parsedIP := net.ParseIP(host)
if (rejectIP && parsedIP != nil) ||
(rejectDomain && parsedIP == nil) {
continue
}
hosts = append(hosts, host)
}
return hosts
}

66
internal/updater/openvpn/extract.go Normal file
View File

@@ -0,0 +1,66 @@
package openvpn
import (
"errors"
"fmt"
"net"
"strings"
)
var (
ErrNoRemoteHost = errors.New("remote host not found")
ErrNoRemoteIP = errors.New("remote IP not found")
)
func ExtractHost(b []byte) (host, warning string, err error) {
const (
rejectIP = true
rejectDomain = false
)
hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", "", ErrNoRemoteHost
} else if len(hosts) > 1 {
warning = fmt.Sprintf(
"only using the first host %q and discarding %d other hosts",
hosts[0], len(hosts)-1)
}
return hosts[0], warning, nil
}
func ExtractIP(b []byte) (ip net.IP, warning string, err error) {
const (
rejectIP = false
rejectDomain = true
)
ips := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(ips) == 0 {
return nil, "", ErrNoRemoteIP
} else if len(ips) > 1 {
warning = fmt.Sprintf(
"only using the first IP address %s and discarding %d other hosts",
ips[0], len(ips)-1)
}
return net.ParseIP(ips[0]), warning, nil
}
func extractRemoteHosts(content []byte, rejectIP, rejectDomain bool) (hosts []string) {
lines := strings.Split(string(content), "\n")
for _, line := range lines {
if !strings.HasPrefix(line, "remote ") {
continue
}
fields := strings.Fields(line)
if len(fields) == 1 || len(fields[1]) == 0 {
continue
}
host := fields[1]
parsedIP := net.ParseIP(host)
if (rejectIP && parsedIP != nil) ||
(rejectDomain && parsedIP == nil) {
continue
}
hosts = append(hosts, host)
}
return hosts
}

41
internal/updater/openvpn/fetch.go Normal file
View File

@@ -0,0 +1,41 @@
package openvpn
import (
"context"
"fmt"
"io"
"net/http"
)
func FetchFile(ctx context.Context, client *http.Client, url string) (
host string, err error) {
b, err := fetchData(ctx, client, url)
if err != nil {
return "", err
}
const rejectIP = true
const rejectDomain = false
hosts := extractRemoteHosts(b, rejectIP, rejectDomain)
if len(hosts) == 0 {
return "", fmt.Errorf("%w for url %s", ErrNoRemoteHost, url)
}
return hosts[0], nil
}
func fetchData(ctx context.Context, client *http.Client, url string) (
b []byte, err error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
return io.ReadAll(response.Body)
}

59
internal/updater/openvpn/multifetch.go Normal file
View File

@@ -0,0 +1,59 @@
package openvpn
import (
"context"
"net/http"
)
// FetchMultiFiles fetches multiple Openvpn files in parallel and
// parses them to extract each of their host. A mapping from host to
// URL is returned.
func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string) (
hostToURL map[string]string, err error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
hostToURL = make(map[string]string, len(urls))
type Result struct {
url string
host string
}
results := make(chan Result)
defer close(results)
errors := make(chan error)
defer close(errors)
for _, url := range urls {
go func(url string) {
host, err := FetchFile(ctx, client, url)
if err != nil {
errors <- err
return
}
results <- Result{
url: url,
host: host,
}
}(url)
}
for range urls {
select {
case newErr := <-errors:
if err == nil { // only assign to the first error
err = newErr
cancel() // stop other operations, this will trigger other errors we ignore
}
case result := <-results:
hostToURL[result.host] = result.url
}
}
if err != nil {
return nil, err
}
return hostToURL, nil
}

145
internal/updater/pia.go
View File

@@ -1,145 +0,0 @@
package updater
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func (u *updater) updatePIA(ctx context.Context) (err error) {
const url = "https://serverlist.piaservers.net/vpninfo/servers/v5"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
response, err := u.client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url)
}
b, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
if err := response.Body.Close(); err != nil {
return err
}
// remove key/signature at the bottom
i := bytes.IndexRune(b, '\n')
b = b[:i]
var data struct {
Regions []struct {
Name string `json:"name"`
PortForward bool `json:"port_forward"`
Servers struct {
UDP []struct {
IP net.IP `json:"ip"`
CN string `json:"cn"`
} `json:"ovpnudp"`
TCP []struct {
IP net.IP `json:"ip"`
CN string `json:"cn"`
} `json:"ovpntcp"`
} `json:"servers"`
} `json:"regions"`
}
if err := json.Unmarshal(b, &data); err != nil {
return err
}
// Deduplicate servers with the same IP address
type NetProtocols struct {
tcp, udp bool
}
ipToProtocols := make(map[string]NetProtocols)
for _, region := range data.Regions {
for _, udpServer := range region.Servers.UDP {
protocols := ipToProtocols[udpServer.IP.String()]
protocols.udp = true
ipToProtocols[udpServer.IP.String()] = protocols
}
for _, tcpServer := range region.Servers.TCP {
protocols := ipToProtocols[tcpServer.IP.String()]
protocols.tcp = true
ipToProtocols[tcpServer.IP.String()] = protocols
}
}
servers := make([]models.PIAServer, 0, len(ipToProtocols)) // set the capacity, not the length of the slice
for _, region := range data.Regions {
for _, udpServer := range region.Servers.UDP {
protocols, ok := ipToProtocols[udpServer.IP.String()]
if !ok { // already added that IP for a server
continue
}
server := models.PIAServer{
Region: region.Name,
ServerName: udpServer.CN,
TCP: protocols.tcp,
UDP: protocols.udp,
PortForward: region.PortForward,
IP: udpServer.IP,
}
delete(ipToProtocols, udpServer.IP.String())
servers = append(servers, server)
}
for _, tcpServer := range region.Servers.TCP {
protocols, ok := ipToProtocols[tcpServer.IP.String()]
if !ok { // already added that IP for a server
continue
}
server := models.PIAServer{
Region: region.Name,
ServerName: tcpServer.CN,
TCP: protocols.tcp,
UDP: protocols.udp,
PortForward: region.PortForward,
IP: tcpServer.IP,
}
delete(ipToProtocols, tcpServer.IP.String())
servers = append(servers, server)
}
}
sort.Slice(servers, func(i, j int) bool {
if servers[i].Region == servers[j].Region {
return servers[i].ServerName < servers[j].ServerName
}
return servers[i].Region < servers[j].Region
})
if u.options.Stdout {
u.println(stringifyPIAServers(servers))
}
u.servers.Pia.Timestamp = u.timeNow().Unix()
u.servers.Pia.Servers = servers
return nil
}
func stringifyPIAServers(servers []models.PIAServer) (s string) {
s = "func PIAServers() []models.PIAServer {\n"
s += " return []models.PIAServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

106
internal/updater/privado.go
View File

@@ -1,106 +0,0 @@
package updater
import (
"context"
"fmt"
"net/http"
"sort"
"time"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updatePrivado(ctx context.Context) (err error) {
servers, warnings, err := findPrivadoServersFromZip(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Privado: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Privado servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyPrivadoServers(servers))
}
u.servers.Privado.Timestamp = u.timeNow().Unix()
u.servers.Privado.Servers = servers
return nil
}
func findPrivadoServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.PrivadoServer, warnings []string, err error) {
const zipURL = "https://privado.io/apps/ovpn_configs.zip"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
hosts := make([]string, 0, len(contents))
for fileName, content := range contents {
hostname, warning, err := extractHostFromOVPN(content)
if len(warning) > 0 {
warnings = append(warnings, warning)
}
if err != nil {
return nil, warnings, fmt.Errorf("%w in %q", err, fileName)
}
hosts = append(hosts, hostname)
}
const (
maxFailRatio = 0.1
maxDuration = 3 * time.Second
maxNoNew = 1
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
for hostname, IPs := range hostToIPs {
switch len(IPs) {
case 0:
warning := fmt.Sprintf("no IP address found for host %q", hostname)
warnings = append(warnings, warning)
continue
case 1:
default:
warning := fmt.Sprintf("more than one IP address found for host %q", hostname)
warnings = append(warnings, warning)
}
server := models.PrivadoServer{
Hostname: hostname,
IP: IPs[0],
}
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].Hostname < servers[j].Hostname
})
return servers, warnings, nil
}
func stringifyPrivadoServers(servers []models.PrivadoServer) (s string) {
s = "func PrivadoServers() []models.PrivadoServer {\n"
s += " return []models.PrivadoServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

151
internal/updater/privatevpn.go
View File

@@ -1,151 +0,0 @@
package updater
import (
"context"
"fmt"
"net/http"
"regexp"
"sort"
"strings"
"time"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updatePrivatevpn(ctx context.Context) (err error) {
servers, warnings, err := findPrivatevpnServersFromZip(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Privatevpn: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Privatevpn servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyPrivatevpnServers(servers))
}
u.servers.Privatevpn.Timestamp = u.timeNow().Unix()
u.servers.Privatevpn.Servers = servers
return nil
}
func findPrivatevpnServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.PrivatevpnServer, warnings []string, err error) {
// Note: all servers do both TCP and UDP
const zipURL = "https://privatevpn.com/client/PrivateVPN-TUN.zip"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
trailingNumber := regexp.MustCompile(` [0-9]+$`)
countryCodes := constants.CountryCodes()
uniqueServers := map[string]models.PrivatevpnServer{} // key is the hostname
for fileName, content := range contents {
const prefix = "PrivateVPN-"
const suffix = "-TUN-443.ovpn"
if !strings.HasSuffix(fileName, suffix) {
continue // only process TCP servers as they're the same
}
var server models.PrivatevpnServer
s := strings.TrimPrefix(fileName, prefix)
s = strings.TrimSuffix(s, suffix)
s = trailingNumber.ReplaceAllString(s, "")
parts := strings.Split(s, "-")
var countryCode string
countryCode, server.City = parts[0], parts[1]
countryCode = strings.ToLower(countryCode)
var countryCodeOK bool
server.Country, countryCodeOK = countryCodes[countryCode]
if !countryCodeOK {
warnings = append(warnings, "unknown country code: "+countryCode)
server.Country = countryCode
}
var warning string
server.Hostname, warning, err = extractHostFromOVPN(content)
if len(warning) > 0 {
warnings = append(warnings, warning)
}
if err != nil {
return nil, warnings, err
}
if len(warning) > 0 {
continue
}
uniqueServers[server.Hostname] = server
}
hostnames := make([]string, len(uniqueServers))
i := 0
for hostname := range uniqueServers {
hostnames[i] = hostname
i++
}
const (
maxFailRatio = 0.1
maxDuration = 6 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, newWarnings, err := presolver.Resolve(ctx, hostnames, settings)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
for hostname, server := range uniqueServers {
ips := hostToIPs[hostname]
if len(ips) == 0 {
continue
}
server.IPs = ips
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Country < servers[j].Country
})
return servers, warnings, nil
}
func stringifyPrivatevpnServers(servers []models.PrivatevpnServer) (s string) {
s = "func PrivatevpnServers() []models.PrivatevpnServer {\n"
s += " return []models.PrivatevpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

141
internal/updater/protonvpn.go
View File

@@ -1,141 +0,0 @@
package updater
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"sort"
"strings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
)
func (u *updater) updateProtonvpn(ctx context.Context) (err error) {
servers, warnings, err := findProtonvpnServers(ctx, u.client)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Protonvpn: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Protonvpn servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyProtonvpnServers(servers))
}
u.servers.Protonvpn.Timestamp = u.timeNow().Unix()
u.servers.Protonvpn.Servers = servers
return nil
}
func findProtonvpnServers(ctx context.Context, client *http.Client) (
servers []models.ProtonvpnServer, warnings []string, err error) {
const url = "https://api.protonmail.ch/vpn/logicals"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url)
}
decoder := json.NewDecoder(response.Body)
var data struct {
LogicalServers []struct {
Name string
ExitCountry string
Region *string
City *string
Servers []struct {
EntryIP net.IP
ExitIP net.IP
Domain string
Status uint8
}
}
}
if err := decoder.Decode(&data); err != nil {
return nil, nil, err
}
if err := response.Body.Close(); err != nil {
return nil, nil, err
}
countryCodesMapping := constants.CountryCodes()
for _, logicalServer := range data.LogicalServers {
for _, physicalServer := range logicalServer.Servers {
if physicalServer.Status == 0 {
warnings = append(warnings, "ignoring server "+physicalServer.Domain+" as its status is 0")
continue
}
countryCode := strings.ToLower(logicalServer.ExitCountry)
country, ok := countryCodesMapping[countryCode]
if !ok {
warnings = append(warnings, "country not found for country code "+countryCode)
country = logicalServer.ExitCountry
}
server := models.ProtonvpnServer{
// Note: for multi-hop use the server name or hostname instead of the country
Country: country,
Region: getStringValue(logicalServer.Region),
City: getStringValue(logicalServer.City),
Name: logicalServer.Name,
Hostname: physicalServer.Domain,
EntryIP: physicalServer.EntryIP,
ExitIP: physicalServer.ExitIP,
}
servers = append(servers, server)
}
}
sort.Slice(servers, func(i, j int) bool {
a, b := servers[i], servers[j]
if a.Country == b.Country { //nolint:nestif
if a.Region == b.Region {
if a.City == b.City {
if a.Name == b.Name {
return a.Hostname < b.Hostname
}
return a.Name < b.Name
}
return a.City < b.City
}
return a.Region < b.Region
}
return a.Country < b.Country
})
return servers, warnings, nil
}
func getStringValue(ptr *string) string {
if ptr == nil {
return ""
}
return *ptr
}
func stringifyProtonvpnServers(servers []models.ProtonvpnServer) (s string) {
s = "func ProtonvpnServers() []models.ProtonvpnServer {\n"
s += " return []models.ProtonvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

279
internal/updater/providers.go Normal file
View File

@@ -0,0 +1,279 @@
package updater
import (
"context"
"fmt"
"github.com/qdm12/gluetun/internal/updater/providers/cyberghost"
"github.com/qdm12/gluetun/internal/updater/providers/fastestvpn"
"github.com/qdm12/gluetun/internal/updater/providers/hidemyass"
"github.com/qdm12/gluetun/internal/updater/providers/mullvad"
"github.com/qdm12/gluetun/internal/updater/providers/nordvpn"
"github.com/qdm12/gluetun/internal/updater/providers/pia"
"github.com/qdm12/gluetun/internal/updater/providers/privado"
"github.com/qdm12/gluetun/internal/updater/providers/privatevpn"
"github.com/qdm12/gluetun/internal/updater/providers/protonvpn"
"github.com/qdm12/gluetun/internal/updater/providers/purevpn"
"github.com/qdm12/gluetun/internal/updater/providers/surfshark"
"github.com/qdm12/gluetun/internal/updater/providers/torguard"
"github.com/qdm12/gluetun/internal/updater/providers/vyprvpn"
"github.com/qdm12/gluetun/internal/updater/providers/windscribe"
)
func (u *updater) updateCyberghost(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Cyberghost.Servers))
servers, err := cyberghost.GetServers(ctx, u.presolver, minServers)
if err != nil {
return err
}
if u.options.Stdout {
u.println(cyberghost.Stringify(servers))
}
u.servers.Cyberghost.Timestamp = u.timeNow().Unix()
u.servers.Cyberghost.Servers = servers
return nil
}
func (u *updater) updateFastestvpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Fastestvpn.Servers))
servers, warnings, err := fastestvpn.GetServers(
ctx, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("FastestVPN: " + warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(fastestvpn.Stringify(servers))
}
u.servers.Fastestvpn.Timestamp = u.timeNow().Unix()
u.servers.Fastestvpn.Servers = servers
return nil
}
func (u *updater) updateHideMyAss(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.HideMyAss.Servers))
servers, warnings, err := hidemyass.GetServers(
ctx, u.client, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("HideMyAss: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(hidemyass.Stringify(servers))
}
u.servers.HideMyAss.Timestamp = u.timeNow().Unix()
u.servers.HideMyAss.Servers = servers
return nil
}
func (u *updater) updateMullvad(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Mullvad.Servers))
servers, err := mullvad.GetServers(ctx, u.client, minServers)
if err != nil {
return err
}
if u.options.Stdout {
u.println(mullvad.Stringify(servers))
}
u.servers.Mullvad.Timestamp = u.timeNow().Unix()
u.servers.Mullvad.Servers = servers
return nil
}
func (u *updater) updateNordvpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Nordvpn.Servers))
servers, warnings, err := nordvpn.GetServers(ctx, u.client, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("NordVPN: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(nordvpn.Stringify(servers))
}
u.servers.Nordvpn.Timestamp = u.timeNow().Unix()
u.servers.Nordvpn.Servers = servers
return nil
}
func (u *updater) updatePIA(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Pia.Servers))
servers, err := pia.GetServers(ctx, u.client, minServers)
if err != nil {
return err
}
if u.options.Stdout {
u.println(pia.Stringify(servers))
}
u.servers.Pia.Timestamp = u.timeNow().Unix()
u.servers.Pia.Servers = servers
return nil
}
func (u *updater) updatePrivado(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Privado.Servers))
servers, warnings, err := privado.GetServers(
ctx, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Privado: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(privado.Stringify(servers))
}
u.servers.Privado.Timestamp = u.timeNow().Unix()
u.servers.Privado.Servers = servers
return nil
}
func (u *updater) updatePrivatevpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Privatevpn.Servers))
servers, warnings, err := privatevpn.GetServers(
ctx, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("PrivateVPN: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(privatevpn.Stringify(servers))
}
u.servers.Privatevpn.Timestamp = u.timeNow().Unix()
u.servers.Privatevpn.Servers = servers
return nil
}
func (u *updater) updateProtonvpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Privatevpn.Servers))
servers, warnings, err := protonvpn.GetServers(ctx, u.client, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("ProtonVPN: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(protonvpn.Stringify(servers))
}
u.servers.Protonvpn.Timestamp = u.timeNow().Unix()
u.servers.Protonvpn.Servers = servers
return nil
}
func (u *updater) updatePurevpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Purevpn.Servers))
servers, warnings, err := purevpn.GetServers(
ctx, u.client, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("PureVPN: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Purevpn servers: %w", err)
}
if u.options.Stdout {
u.println(purevpn.Stringify(servers))
}
u.servers.Purevpn.Timestamp = u.timeNow().Unix()
u.servers.Purevpn.Servers = servers
return nil
}
func (u *updater) updateSurfshark(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Surfshark.Servers))
servers, warnings, err := surfshark.GetServers(
ctx, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Surfshark: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(surfshark.Stringify(servers))
}
u.servers.Surfshark.Timestamp = u.timeNow().Unix()
u.servers.Surfshark.Servers = servers
return nil
}
func (u *updater) updateTorguard(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Torguard.Servers))
servers, warnings, err := torguard.GetServers(ctx, u.unzipper, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Torguard: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(torguard.Stringify(servers))
}
u.servers.Torguard.Timestamp = u.timeNow().Unix()
u.servers.Torguard.Servers = servers
return nil
}
func (u *updater) updateVyprvpn(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Vyprvpn.Servers))
servers, warnings, err := vyprvpn.GetServers(
ctx, u.unzipper, u.presolver, minServers)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("VyprVPN: %s", warning)
}
}
if err != nil {
return err
}
if u.options.Stdout {
u.println(vyprvpn.Stringify(servers))
}
u.servers.Vyprvpn.Timestamp = u.timeNow().Unix()
u.servers.Vyprvpn.Servers = servers
return nil
}
func (u *updater) updateWindscribe(ctx context.Context) (err error) {
minServers := getMinServers(len(u.servers.Windscribe.Servers))
servers, err := windscribe.GetServers(ctx, u.client, minServers)
if err != nil {
return err
}
if u.options.Stdout {
u.println(windscribe.Stringify(servers))
}
u.servers.Windscribe.Timestamp = u.timeNow().Unix()
u.servers.Windscribe.Servers = servers
return nil
}
func getMinServers(existingServers int) (minServers int) {
const minRatio = 0.8
return int(minRatio * float64(existingServers))
}

131
internal/updater/cyberghost.go → internal/updater/providers/cyberghost/constants.go
View File

@@ -1,117 +1,6 @@
package updater package cyberghost
import ( func getGroups() map[string]string {
"context"
"sort"
"time"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updateCyberghost(ctx context.Context) (err error) {
servers, err := findCyberghostServers(ctx, u.presolver)
if err != nil {
return err
}
if u.options.Stdout {
u.println(stringifyCyberghostServers(servers))
}
u.servers.Cyberghost.Timestamp = u.timeNow().Unix()
u.servers.Cyberghost.Servers = servers
return nil
}
func findCyberghostServers(ctx context.Context, presolver resolver.Parallel) (
servers []models.CyberghostServer, err error) {
groups := getCyberghostGroups()
allCountryCodes := constants.CountryCodes()
cyberghostCountryCodes := getCyberghostSubdomainToRegion()
possibleCountryCodes := mergeCountryCodes(cyberghostCountryCodes, allCountryCodes)
// key is the host
possibleServers := make(map[string]models.CyberghostServer, len(groups)*len(possibleCountryCodes))
possibleHosts := make([]string, 0, len(groups)*len(possibleCountryCodes))
for groupID, groupName := range groups {
for countryCode, region := range possibleCountryCodes {
const domain = "cg-dialup.net"
possibleHost := groupID + "-" + countryCode + "." + domain
possibleHosts = append(possibleHosts, possibleHost)
possibleServer := models.CyberghostServer{
Region: region,
Group: groupName,
}
possibleServers[possibleHost] = possibleServer
}
}
const (
maxFailRatio = 1
minFound = 100
maxDuration = 10 * time.Second
maxNoNew = 2
maxFails = 1
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minFound,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: time.Second,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, _, err := presolver.Resolve(ctx, possibleHosts, settings)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
// Set IPs for servers found
for host, IPs := range hostToIPs {
server := possibleServers[host]
server.IPs = IPs
possibleServers[host] = server
}
// Remove servers with no IPs (aka not found)
for host, server := range possibleServers {
if len(server.IPs) == 0 {
delete(possibleServers, host)
}
}
// Flatten possibleServers to a slice
servers = make([]models.CyberghostServer, 0, len(possibleServers))
for _, server := range possibleServers {
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].Region < servers[j].Region
})
return servers, nil
}
//nolint:goconst
func stringifyCyberghostServers(servers []models.CyberghostServer) (s string) {
s = "func CyberghostServers() []models.CyberghostServer {\n"
s += " return []models.CyberghostServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}
func getCyberghostGroups() map[string]string {
return map[string]string{ return map[string]string{
"87-1": "Premium UDP Europe", "87-1": "Premium UDP Europe",
"94-1": "Premium UDP USA", "94-1": "Premium UDP USA",
@@ -124,21 +13,7 @@ func getCyberghostGroups() map[string]string {
} }
} }
func mergeCountryCodes(base, extend map[string]string) (merged map[string]string) { func getSubdomainToRegion() map[string]string {
merged = make(map[string]string, len(base))
for countryCode, region := range base {
merged[countryCode] = region
}
for countryCode := range base {
delete(extend, countryCode)
}
for countryCode, region := range extend {
merged[countryCode] = region
}
return merged
}
func getCyberghostSubdomainToRegion() map[string]string {
return map[string]string{ return map[string]string{
"af": "Afghanistan", "af": "Afghanistan",
"ax": "Aland Islands", "ax": "Aland Islands",

15
internal/updater/providers/cyberghost/countries.go Normal file
View File

@@ -0,0 +1,15 @@
package cyberghost
func mergeCountryCodes(base, extend map[string]string) (merged map[string]string) {
merged = make(map[string]string, len(base))
for countryCode, region := range base {
merged[countryCode] = region
}
for countryCode := range base {
delete(extend, countryCode)
}
for countryCode, region := range extend {
merged[countryCode] = region
}
return merged
}

65
internal/updater/providers/cyberghost/hosttoserver.go Normal file
View File

@@ -0,0 +1,65 @@
package cyberghost
import (
"net"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.CyberghostServer
func getPossibleServers() (possibleServers hostToServer) {
groups := getGroups()
cyberghostCountryCodes := getSubdomainToRegion()
allCountryCodes := constants.CountryCodes()
possibleCountryCodes := mergeCountryCodes(cyberghostCountryCodes, allCountryCodes)
n := len(groups) * len(possibleCountryCodes)
possibleServers = make(hostToServer, n) // key is the host
for groupID, groupName := range groups {
for countryCode, region := range possibleCountryCodes {
const domain = "cg-dialup.net"
possibleHost := groupID + "-" + countryCode + "." + domain
possibleServer := models.CyberghostServer{
Region: region,
Group: groupName,
}
possibleServers[possibleHost] = possibleServer
}
}
return possibleServers
}
func (hts hostToServer) hostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toSlice() (servers []models.CyberghostServer) {
servers = make([]models.CyberghostServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

42
internal/updater/providers/cyberghost/resolve.go Normal file
View File

@@ -0,0 +1,42 @@
package cyberghost
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
possibleHosts []string, minServers int) (
hostToIPs map[string][]net.IP, err error) {
const (
maxFailRatio = 1
maxDuration = 10 * time.Second
betweenDuration = 500 * time.Millisecond
maxNoNew = 2
maxFails = 10
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, _, err = presolver.Resolve(ctx, possibleHosts, settings)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
return hostToIPs, nil
}

28
internal/updater/providers/cyberghost/servers.go Normal file
View File

@@ -0,0 +1,28 @@
// Package cyberghost contains code to obtain the server information
// for the Cyberghost provider.
package cyberghost
import (
"context"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func GetServers(ctx context.Context, presolver resolver.Parallel,
minServers int) (servers []models.CyberghostServer, err error) {
possibleServers := getPossibleServers()
possibleHosts := possibleServers.hostsSlice()
hostToIPs, err := resolveHosts(ctx, presolver, possibleHosts, minServers)
if err != nil {
return nil, err
}
possibleServers.adaptWithIPs(hostToIPs)
servers = possibleServers.toSlice()
sortServers(servers)
return servers, nil
}

16
internal/updater/providers/cyberghost/sort.go Normal file
View File

@@ -0,0 +1,16 @@
package cyberghost
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.CyberghostServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Region == servers[j].Region {
return servers[i].Group < servers[j].Group
}
return servers[i].Region < servers[j].Region
})
}

15
internal/updater/providers/cyberghost/string.go Normal file
View File

@@ -0,0 +1,15 @@
package cyberghost
import "github.com/qdm12/gluetun/internal/models"
// Stringify converts servers to code string format.
func Stringify(servers []models.CyberghostServer) (s string) {
s = "func CyberghostServers() []models.CyberghostServer {\n"
s += " return []models.CyberghostServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

39
internal/updater/providers/fastestvpn/filename.go Normal file
View File

@@ -0,0 +1,39 @@
package fastestvpn
import (
"errors"
"fmt"
"regexp"
"strings"
)
var errFilenameNoProtocolSuffix = errors.New("filename does not have a protocol suffix")
var trailNumberExp = regexp.MustCompile(`[0-9]+$`)
func parseFilename(fileName string) (
country string, tcp, udp bool, err error,
) {
const (
tcpSuffix = "-TCP.ovpn"
udpSuffix = "-UDP.ovpn"
)
var suffix string
switch {
case strings.HasSuffix(fileName, tcpSuffix):
suffix = tcpSuffix
tcp = true
case strings.HasSuffix(fileName, udpSuffix):
suffix = udpSuffix
udp = true
default:
return "", false, false, fmt.Errorf("%w: %s",
errFilenameNoProtocolSuffix, fileName)
}
countryWithNumber := strings.TrimSuffix(fileName, suffix)
number := trailNumberExp.FindString(countryWithNumber)
country = countryWithNumber[:len(countryWithNumber)-len(number)]
return country, tcp, udp, nil
}

53
internal/updater/providers/fastestvpn/hosttoserver.go Normal file
View File

@@ -0,0 +1,53 @@
package fastestvpn
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.FastestvpnServer
func (hts hostToServer) add(host, country string, tcp, udp bool) {
server, ok := hts[host]
if !ok {
server.Hostname = host
server.Country = country
}
if tcp {
server.TCP = true
}
if udp {
server.UDP = true
}
hts[host] = server
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.FastestvpnServer) {
servers = make([]models.FastestvpnServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

30
internal/updater/providers/fastestvpn/resolve.go Normal file
View File

@@ -0,0 +1,30 @@
package fastestvpn
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxNoNew = 1
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: time.Second,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

82
internal/updater/providers/fastestvpn/servers.go Normal file
View File

@@ -0,0 +1,82 @@
// Package fastestvpn contains code to obtain the server information
// for the FastestVPN provider.
package fastestvpn
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
presolver resolver.Parallel, minServers int) (
servers []models.FastestvpnServer, warnings []string, err error) {
const url = "https://support.fastestvpn.com/download/openvpn-tcp-udp-config-files"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
country, tcp, udp, err := parseFilename(fileName)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
hts.add(host, country, tcp, udp)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = hts.toServersSlice()
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}

16
internal/updater/providers/fastestvpn/sort.go Normal file
View File

@@ -0,0 +1,16 @@
package fastestvpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.FastestvpnServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/fastestvpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package fastestvpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.FastestvpnServer) (s string) {
s = "func FastestvpnServers() []models.FastestvpnServer {\n"
s += " return []models.FastestvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

19
internal/updater/providers/hidemyass/hosts.go Normal file
View File

@@ -0,0 +1,19 @@
package hidemyass
func getUniqueHosts(tcpHostToURL, udpHostToURL map[string]string) (
hosts []string) {
uniqueHosts := make(map[string]struct{}, len(tcpHostToURL))
for host := range tcpHostToURL {
uniqueHosts[host] = struct{}{}
}
for host := range udpHostToURL {
uniqueHosts[host] = struct{}{}
}
hosts = make([]string, 0, len(uniqueHosts))
for host := range uniqueHosts {
hosts = append(hosts, host)
}
return hosts
}

37
internal/updater/providers/hidemyass/hosttourl.go Normal file
View File

@@ -0,0 +1,37 @@
package hidemyass
import (
"context"
"net/http"
"strings"
"github.com/qdm12/gluetun/internal/updater/openvpn"
)
func getAllHostToURL(ctx context.Context, client *http.Client) (
tcpHostToURL, udpHostToURL map[string]string, err error) {
tcpHostToURL, err = getHostToURL(ctx, client, "TCP")
if err != nil {
return nil, nil, err
}
udpHostToURL, err = getHostToURL(ctx, client, "UDP")
if err != nil {
return nil, nil, err
}
return tcpHostToURL, udpHostToURL, nil
}
func getHostToURL(ctx context.Context, client *http.Client, protocol string) (
hostToURL map[string]string, err error) {
const baseURL = "https://vpn.hidemyass.com/vpn-config"
indexURL := baseURL + "/" + strings.ToUpper(protocol) + "/"
urls, err := fetchIndex(ctx, client, indexURL)
if err != nil {
return nil, err
}
return openvpn.FetchMultiFiles(ctx, client, urls)
}

54
internal/updater/providers/hidemyass/index.go Normal file
View File

@@ -0,0 +1,54 @@
package hidemyass
import (
"context"
"io"
"net/http"
"regexp"
"strings"
)
var indexOpenvpnLinksRegex = regexp.MustCompile(`<a[ ]+href=".+\.ovpn">.+\.ovpn</a>`)
func fetchIndex(ctx context.Context, client *http.Client, indexURL string) (
openvpnURLs []string, err error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
htmlCode, err := io.ReadAll(response.Body)
if err != nil {
return nil, err
}
if !strings.HasSuffix(indexURL, "/") {
indexURL += "/"
}
lines := strings.Split(string(htmlCode), "\n")
for _, line := range lines {
found := indexOpenvpnLinksRegex.FindString(line)
if len(found) == 0 {
continue
}
const prefix = `.ovpn">`
const suffix = `</a>`
startIndex := strings.Index(found, prefix) + len(prefix)
endIndex := strings.Index(found, suffix)
filename := found[startIndex:endIndex]
openvpnURL := indexURL + filename
if !strings.HasSuffix(openvpnURL, ".ovpn") {
continue
}
openvpnURLs = append(openvpnURLs, openvpnURL)
}
return openvpnURLs, nil
}

33
internal/updater/providers/hidemyass/resolve.go Normal file
View File

@@ -0,0 +1,33 @@
package hidemyass
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (
hostToIPs map[string][]net.IP, warnings []string, err error) {
const (
maxFailRatio = 0.1
maxDuration = 15 * time.Second
betweenDuration = 2 * time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

68
internal/updater/providers/hidemyass/servers.go Normal file
View File

@@ -0,0 +1,68 @@
// Package hidemyass contains code to obtain the server information
// for the HideMyAss provider.
package hidemyass
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client,
presolver resolver.Parallel, minServers int) (
servers []models.HideMyAssServer, warnings []string, err error) {
tcpHostToURL, udpHostToURL, err := getAllHostToURL(ctx, client)
if err != nil {
return nil, nil, err
}
hosts := getUniqueHosts(tcpHostToURL, udpHostToURL)
if len(hosts) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hosts), minServers)
}
hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
if err != nil {
return nil, warnings, err
}
servers = make([]models.HideMyAssServer, 0, len(hostToIPs))
for host, IPs := range hostToIPs {
tcpURL, tcp := tcpHostToURL[host]
udpURL, udp := udpHostToURL[host]
// These two are only used to extract the country, region and city.
var url, protocol string
if tcp {
url = tcpURL
protocol = "TCP"
} else if udp {
url = udpURL
protocol = "UDP"
}
country, region, city := parseOpenvpnURL(url, protocol)
server := models.HideMyAssServer{
Country: country,
Region: region,
City: city,
Hostname: host,
IPs: IPs,
TCP: tcp,
UDP: udp,
}
servers = append(servers, server)
}
sortServers(servers)
return servers, warnings, nil
}

22
internal/updater/providers/hidemyass/sort.go Normal file
View File

@@ -0,0 +1,22 @@
package hidemyass
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.HideMyAssServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].Region == servers[j].Region {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Region < servers[j].Region
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/hidemyass/string.go Normal file
View File

@@ -0,0 +1,14 @@
package hidemyass
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.HideMyAssServer) (s string) {
s = "func HideMyAssServers() []models.HideMyAssServer {\n"
s += " return []models.HideMyAssServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

44
internal/updater/providers/hidemyass/url.go Normal file
View File

@@ -0,0 +1,44 @@
package hidemyass
import (
"strings"
"unicode"
)
func parseOpenvpnURL(url, protocol string) (country, region, city string) {
lastSlashIndex := strings.LastIndex(url, "/")
url = url[lastSlashIndex+1:]
suffix := "." + strings.ToUpper(protocol) + ".ovpn"
url = strings.TrimSuffix(url, suffix)
parts := strings.Split(url, ".")
switch len(parts) {
case 1:
country = parts[0]
return country, "", ""
case 2: //nolint:gomnd
country = parts[0]
city = parts[1]
default:
country = parts[0]
region = parts[1]
city = parts[2]
}
return camelCaseToWords(country), camelCaseToWords(region),
camelCaseToWords(city)
}
func camelCaseToWords(camelCase string) (words string) {
wasLowerCase := false
for _, r := range camelCase {
if wasLowerCase && unicode.IsUpper(r) {
words += " "
}
wasLowerCase = unicode.IsLower(r)
words += string(r)
}
return words
}

55
internal/updater/providers/mullvad/api.go Normal file
View File

@@ -0,0 +1,55 @@
package mullvad
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
)
type serverData struct {
Hostname string `json:"hostname"`
Country string `json:"country_name"`
City string `json:"city_name"`
Active bool `json:"active"`
Owned bool `json:"owned"`
Provider string `json:"provider"`
IPv4 string `json:"ipv4_addr_in"`
IPv6 string `json:"ipv6_addr_in"`
}
func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) {
const url = "https://api.mullvad.net/www/relays/openvpn/"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
}
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&data); err != nil {
return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
}
if err := response.Body.Close(); err != nil {
return nil, err
}
return data, nil
}

58
internal/updater/providers/mullvad/hosttoserver.go Normal file
View File

@@ -0,0 +1,58 @@
package mullvad
import (
"errors"
"fmt"
"net"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.MullvadServer
var (
ErrParseIPv4 = errors.New("cannot parse IPv4 address")
ErrParseIPv6 = errors.New("cannot parse IPv6 address")
)
func (hts hostToServer) add(data serverData) (err error) {
if !data.Active {
return
}
ipv4 := net.ParseIP(data.IPv4)
if ipv4 == nil || ipv4.To4() == nil {
return fmt.Errorf("%w: %s", ErrParseIPv4, data.IPv4)
}
ipv6 := net.ParseIP(data.IPv6)
if ipv6 == nil || ipv6.To4() != nil {
return fmt.Errorf("%w: %s", ErrParseIPv6, data.IPv6)
}
server, ok := hts[data.Hostname]
if !ok {
server.Country = data.Country
server.City = strings.ReplaceAll(data.City, ",", "")
server.ISP = data.Provider
server.Owned = data.Owned
}
server.IPs = append(server.IPs, ipv4)
server.IPsV6 = append(server.IPsV6, ipv6)
hts[data.Hostname] = server
return nil
}
func (hts hostToServer) toServersSlice() (servers []models.MullvadServer) {
servers = make([]models.MullvadServer, 0, len(hts))
for _, server := range hts {
server.IPs = uniqueSortedIPs(server.IPs)
server.IPsV6 = uniqueSortedIPs(server.IPsV6)
servers = append(servers, server)
}
return servers
}

5
internal/updater/ips.go → internal/updater/providers/mullvad/ips.go
View File

@@ -1,4 +1,4 @@
package updater package mullvad
import ( import (
"bytes" "bytes"
@@ -12,6 +12,7 @@ func uniqueSortedIPs(ips []net.IP) []net.IP {
key := ip.String() key := ip.String()
uniqueIPs[key] = struct{}{} uniqueIPs[key] = struct{}{}
} }
ips = make([]net.IP, 0, len(uniqueIPs)) ips = make([]net.IP, 0, len(uniqueIPs))
for key := range uniqueIPs { for key := range uniqueIPs {
ip := net.ParseIP(key) ip := net.ParseIP(key)
@@ -20,8 +21,10 @@ func uniqueSortedIPs(ips []net.IP) []net.IP {
} }
ips = append(ips, ip) ips = append(ips, ip)
} }
sort.Slice(ips, func(i, j int) bool { sort.Slice(ips, func(i, j int) bool {
return bytes.Compare(ips[i], ips[j]) < 0 return bytes.Compare(ips[i], ips[j]) < 0
}) })
return ips return ips
} }

2
internal/updater/ips_test.go → internal/updater/providers/mullvad/ips_test.go
View File

@@ -1,4 +1,4 @@
package updater package mullvad
import ( import (
"net" "net"

68
internal/updater/providers/mullvad/servers.go Normal file
View File

@@ -0,0 +1,68 @@
// Package mullvad contains code to obtain the server information
// for the Mullvad provider.
package mullvad
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
"github.com/qdm12/gluetun/internal/models"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client, minServers int) (
servers []models.MullvadServer, err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, err
}
hts := make(hostToServer)
for _, serverData := range data {
if err := hts.add(serverData); err != nil {
return nil, err
}
}
if len(hts) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
servers = hts.toServersSlice()
servers = groupByProperties(servers)
sortServers(servers)
return servers, nil
}
// TODO group by hostname so remove this.
func groupByProperties(serversByHost []models.MullvadServer) (serversByProps []models.MullvadServer) {
propsToServer := make(map[string]models.MullvadServer, len(serversByHost))
for _, server := range serversByHost {
key := server.Country + server.City + server.ISP + strconv.FormatBool(server.Owned)
serverByProps, ok := propsToServer[key]
if !ok {
serverByProps.Country = server.Country
serverByProps.City = server.City
serverByProps.ISP = server.ISP
serverByProps.Owned = server.Owned
}
serverByProps.IPs = append(serverByProps.IPs, server.IPs...)
serverByProps.IPsV6 = append(serverByProps.IPsV6, server.IPsV6...)
propsToServer[key] = serverByProps
}
serversByProps = make([]models.MullvadServer, 0, len(propsToServer))
for _, serverByProp := range propsToServer {
serversByProps = append(serversByProps, serverByProp)
}
return serversByProps
}

19
internal/updater/providers/mullvad/sort.go Normal file
View File

@@ -0,0 +1,19 @@
package mullvad
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.MullvadServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].City == servers[j].City {
return servers[i].ISP < servers[j].ISP
}
return servers[i].City < servers[j].City
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/mullvad/string.go Normal file
View File

@@ -0,0 +1,14 @@
package mullvad
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.MullvadServer) (s string) {
s = "func MullvadServers() []models.MullvadServer {\n"
s += " return []models.MullvadServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

6
internal/updater/mullvad_test.go → internal/updater/providers/mullvad/string_test.go
View File

@@ -1,4 +1,4 @@
package updater package mullvad
import ( import (
"net" "net"
@@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func Test_stringifyMullvadServers(t *testing.T) { func Test_Stringify(t *testing.T) {
servers := []models.MullvadServer{{ servers := []models.MullvadServer{{
Country: "webland", Country: "webland",
City: "webcity", City: "webcity",
@@ -27,6 +27,6 @@ func MullvadServers() []models.MullvadServer {
} }
` `
expected = strings.TrimPrefix(strings.TrimSuffix(expected, "\n"), "\n") expected = strings.TrimPrefix(strings.TrimSuffix(expected, "\n"), "\n")
s := stringifyMullvadServers(servers) s := Stringify(servers)
assert.Equal(t, expected, s) assert.Equal(t, expected, s)
} }

55
internal/updater/providers/nordvpn/api.go Normal file
View File

@@ -0,0 +1,55 @@
package nordvpn
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
)
type serverData struct {
Domain string `json:"domain"`
IPAddress string `json:"ip_address"`
Name string `json:"name"`
Country string `json:"country"`
Features struct {
UDP bool `json:"openvpn_udp"`
TCP bool `json:"openvpn_tcp"`
} `json:"features"`
}
func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) {
const url = "https://nordvpn.com/api/server"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
}
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&data); err != nil {
return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
}
if err := response.Body.Close(); err != nil {
return nil, err
}
return data, nil
}

16
internal/updater/providers/nordvpn/ip.go Normal file
View File

@@ -0,0 +1,16 @@
package nordvpn
import (
"fmt"
"net"
)
func parseIPv4(s string) (ipv4 net.IP, err error) {
ip := net.ParseIP(s)
if ip == nil {
return nil, fmt.Errorf("%w: %q", ErrParseIP, s)
} else if ip.To4() == nil {
return nil, fmt.Errorf("%w: %s", ErrNotIPv4, ip)
}
return ip, nil
}

29
internal/updater/providers/nordvpn/name.go Normal file
View File

@@ -0,0 +1,29 @@
package nordvpn
import (
"errors"
"fmt"
"strconv"
"strings"
)
var (
ErrNoIDInServerName = errors.New("no ID in server name")
ErrInvalidIDInServerName = errors.New("invalid ID in server name")
)
func parseServerName(serverName string) (number uint16, err error) {
i := strings.IndexRune(serverName, '#')
if i < 0 {
return 0, fmt.Errorf("%w: %s", ErrNoIDInServerName, serverName)
}
idString := serverName[i+1:]
idUint64, err := strconv.ParseUint(idString, 10, 16)
if err != nil {
return 0, fmt.Errorf("%w: %s", ErrInvalidIDInServerName, serverName)
}
number = uint16(idUint64)
return number, nil
}

64
internal/updater/providers/nordvpn/servers.go Normal file
View File

@@ -0,0 +1,64 @@
// Package nordvpn contains code to obtain the server information
// for the NordVPN provider.
package nordvpn
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/qdm12/gluetun/internal/models"
)
var (
ErrParseIP = errors.New("cannot parse IP address")
ErrNotIPv4 = errors.New("IP address is not IPv4")
ErrNotEnoughServers = errors.New("not enough servers found")
)
func GetServers(ctx context.Context, client *http.Client, minServers int) (
servers []models.NordvpnServer, warnings []string, err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, nil, err
}
servers = make([]models.NordvpnServer, 0, len(data))
for _, jsonServer := range data {
if !jsonServer.Features.TCP && !jsonServer.Features.UDP {
warning := "server does not support TCP and UDP for openvpn: " + jsonServer.Name
warnings = append(warnings, warning)
continue
}
ip, err := parseIPv4(jsonServer.IPAddress)
if err != nil {
return nil, nil, fmt.Errorf("%w for server %s", err, jsonServer.Name)
}
number, err := parseServerName(jsonServer.Name)
if err != nil {
return nil, nil, err
}
server := models.NordvpnServer{
Region: jsonServer.Country,
Number: number,
IP: ip,
TCP: jsonServer.Features.TCP,
UDP: jsonServer.Features.UDP,
}
servers = append(servers, server)
}
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}

16
internal/updater/providers/nordvpn/sort.go Normal file
View File

@@ -0,0 +1,16 @@
package nordvpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.NordvpnServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Region == servers[j].Region {
return servers[i].Number < servers[j].Number
}
return servers[i].Region < servers[j].Region
})
}

14
internal/updater/providers/nordvpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package nordvpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.NordvpnServer) (s string) {
s = "func NordvpnServers() []models.NordvpnServer {\n"
s += " return []models.NordvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

74
internal/updater/providers/pia/api.go Normal file
View File

@@ -0,0 +1,74 @@
package pia
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
)
type apiData struct {
Regions []regionData `json:"regions"`
}
type regionData struct {
Name string `json:"name"`
PortForward bool `json:"port_forward"`
Offline bool `json:"offline"`
Servers struct {
UDP []serverData `json:"ovpnudp"`
TCP []serverData `json:"ovpntcp"`
} `json:"servers"`
}
type serverData struct {
IP net.IP `json:"ip"`
CN string `json:"cn"`
}
func fetchAPI(ctx context.Context, client *http.Client) (
data apiData, err error) {
const url = "https://serverlist.piaservers.net/vpninfo/servers/v5"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return data, err
}
response, err := client.Do(request)
if err != nil {
return data, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return data, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
}
b, err := ioutil.ReadAll(response.Body)
if err != nil {
return data, err
}
if err := response.Body.Close(); err != nil {
return data, err
}
// remove key/signature at the bottom
i := bytes.IndexRune(b, '\n')
b = b[:i]
if err := json.Unmarshal(b, &data); err != nil {
return data, err
}
return data, nil
}

90
internal/updater/providers/pia/servers.go Normal file
View File

@@ -0,0 +1,90 @@
// Package pia contains code to obtain the server information
// for the Private Internet Access provider.
package pia
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/qdm12/gluetun/internal/models"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client, minServers int) (
servers []models.PIAServer, err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, err
}
for _, region := range data.Regions {
// Deduplicate servers with the same common name
commonNameToProtocols := dedupByProtocol(region)
// newServers can support only UDP or both TCP and UDP
newServers := dataToServers(region.Servers.UDP, region.Name,
region.PortForward, commonNameToProtocols)
servers = append(servers, newServers...)
// tcpServers only support TCP as mixed servers were found above.
tcpServers := dataToServers(region.Servers.TCP, region.Name,
region.PortForward, commonNameToProtocols)
servers = append(servers, tcpServers...)
}
if len(servers) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, nil
}
type protocols struct {
tcp bool
udp bool
}
// Deduplicate servers with the same common name for different protocols.
func dedupByProtocol(region regionData) (commonNameToProtocols map[string]protocols) {
commonNameToProtocols = make(map[string]protocols)
for _, udpServer := range region.Servers.UDP {
protocols := commonNameToProtocols[udpServer.CN]
protocols.udp = true
commonNameToProtocols[udpServer.CN] = protocols
}
for _, tcpServer := range region.Servers.TCP {
protocols := commonNameToProtocols[tcpServer.CN]
protocols.tcp = true
commonNameToProtocols[tcpServer.CN] = protocols
}
return commonNameToProtocols
}
func dataToServers(data []serverData, region string, portForward bool,
commonNameToProtocols map[string]protocols) (
servers []models.PIAServer) {
servers = make([]models.PIAServer, 0, len(data))
for _, serverData := range data {
proto, ok := commonNameToProtocols[serverData.CN]
if !ok {
continue // server already added
}
delete(commonNameToProtocols, serverData.CN)
server := models.PIAServer{
Region: region,
ServerName: serverData.CN,
TCP: proto.tcp,
UDP: proto.udp,
PortForward: portForward,
IP: serverData.IP,
}
servers = append(servers, server)
}
return servers
}

16
internal/updater/providers/pia/sort.go Normal file
View File

@@ -0,0 +1,16 @@
package pia
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.PIAServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Region == servers[j].Region {
return servers[i].ServerName < servers[j].ServerName
}
return servers[i].Region < servers[j].Region
})
}

14
internal/updater/providers/pia/string.go Normal file
View File

@@ -0,0 +1,14 @@
package pia
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.PIAServer) (s string) {
s = "func PIAServers() []models.PIAServer {\n"
s += " return []models.PIAServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

53
internal/updater/providers/privado/hosttoserver.go Normal file
View File

@@ -0,0 +1,53 @@
package privado
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.PrivadoServer
func (hts hostToServer) add(host string) {
server, ok := hts[host]
if ok {
return
}
server.Hostname = host
hts[host] = server
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) (
warnings []string) {
for host, IPs := range hostToIPs {
if len(IPs) > 1 {
warning := "more than one IP address found for host " + host
warnings = append(warnings, warning)
}
server := hts[host]
server.IP = IPs[0]
hts[host] = server
}
for host, server := range hts {
if server.IP == nil {
delete(hts, host)
}
}
return warnings
}
func (hts hostToServer) toServersSlice() (servers []models.PrivadoServer) {
servers = make([]models.PrivadoServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

31
internal/updater/providers/privado/resolve.go Normal file
View File

@@ -0,0 +1,31 @@
package privado
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxDuration = 3 * time.Second
maxNoNew = 1
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

72
internal/updater/providers/privado/servers.go Normal file
View File

@@ -0,0 +1,72 @@
// Package privado contains code to obtain the server information
// for the Privado provider.
package privado
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
presolver resolver.Parallel, minServers int) (
servers []models.PrivadoServer, warnings []string, err error) {
const url = "https://privado.io/apps/ovpn_configs.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
hts.add(host)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
newWarnings = hts.adaptWithIPs(hostToIPs)
warnings = append(warnings, newWarnings...)
servers = hts.toServersSlice()
sortServers(servers)
return servers, warnings, nil
}

13
internal/updater/providers/privado/sort.go Normal file
View File

@@ -0,0 +1,13 @@
package privado
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.PrivadoServer) {
sort.Slice(servers, func(i, j int) bool {
return servers[i].Hostname < servers[j].Hostname
})
}

14
internal/updater/providers/privado/string.go Normal file
View File

@@ -0,0 +1,14 @@
package privado
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.PrivadoServer) (s string) {
s = "func PrivadoServers() []models.PrivadoServer {\n"
s += " return []models.PrivadoServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

14
internal/updater/providers/privatevpn/countries.go Normal file
View File

@@ -0,0 +1,14 @@
package privatevpn
import "strings"
func codeToCountry(countryCode string, countryCodes map[string]string) (
country string, warning string) {
countryCode = strings.ToLower(countryCode)
country, ok := countryCodes[countryCode]
if !ok {
warning = "unknown country code: " + countryCode
country = countryCode
}
return country, warning
}

54
internal/updater/providers/privatevpn/filename.go Normal file
View File

@@ -0,0 +1,54 @@
package privatevpn
import (
"errors"
"fmt"
"regexp"
"strings"
)
var (
trailingNumber = regexp.MustCompile(` [0-9]+$`)
)
var (
errBadPrefix = errors.New("bad prefix in file name")
errBadSuffix = errors.New("bad suffix in file name")
errNotEnoughParts = errors.New("not enough parts in file name")
)
func parseFilename(fileName string) (
countryCode, city string, err error,
) {
fileName = strings.ReplaceAll(fileName, " ", "") // remove spaces
const prefix = "PrivateVPN-"
if !strings.HasPrefix(fileName, prefix) {
return "", "", fmt.Errorf("%w: %s", errBadPrefix, fileName)
}
s := strings.TrimPrefix(fileName, prefix)
const tcpSuffix = "-TUN-443.ovpn"
const udpSuffix = "-TUN-1194.ovpn"
switch {
case strings.HasSuffix(fileName, tcpSuffix):
s = strings.TrimSuffix(s, tcpSuffix)
case strings.HasSuffix(fileName, udpSuffix):
s = strings.TrimSuffix(s, udpSuffix)
default:
return "", "", fmt.Errorf("%w: %s", errBadSuffix, fileName)
}
s = trailingNumber.ReplaceAllString(s, "")
parts := strings.Split(s, "-")
const minParts = 2
if len(parts) < minParts {
return "", "", fmt.Errorf("%w: %s",
errNotEnoughParts, fileName)
}
countryCode, city = parts[0], parts[1]
countryCode = strings.ToLower(countryCode)
return countryCode, city, nil
}

50
internal/updater/providers/privatevpn/hosttoserver.go Normal file
View File

@@ -0,0 +1,50 @@
package privatevpn
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.PrivatevpnServer
// TODO check if server supports TCP and UDP.
func (hts hostToServer) add(host, country, city string) {
server, ok := hts[host]
if ok {
return
}
server.Hostname = host
server.Country = country
server.City = city
hts[host] = server
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.PrivatevpnServer) {
servers = make([]models.PrivatevpnServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

33
internal/updater/providers/privatevpn/resolve.go Normal file
View File

@@ -0,0 +1,33 @@
package privatevpn
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxDuration = 6 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

86
internal/updater/providers/privatevpn/servers.go Normal file
View File

@@ -0,0 +1,86 @@
// Package privatevpn contains code to obtain the server information
// for the PrivateVPN provider.
package privatevpn
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
presolver resolver.Parallel, minServers int) (
servers []models.PrivatevpnServer, warnings []string, err error) {
const url = "https://privatevpn.com/client/PrivateVPN-TUN.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
countryCodes := constants.CountryCodes()
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
countryCode, city, err := parseFilename(fileName)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
country, warning := codeToCountry(countryCode, countryCodes)
if warning != "" {
warnings = append(warnings, warning)
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
hts.add(host, country, city)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = hts.toServersSlice()
sortServers(servers)
return servers, warnings, nil
}

19
internal/updater/providers/privatevpn/sort.go Normal file
View File

@@ -0,0 +1,19 @@
package privatevpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.PrivatevpnServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/privatevpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package privatevpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.PrivatevpnServer) (s string) {
s = "func PrivatevpnServers() []models.PrivatevpnServer {\n"
s += " return []models.PrivatevpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

65
internal/updater/providers/protonvpn/api.go Normal file
View File

@@ -0,0 +1,65 @@
package protonvpn
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
)
type apiData struct {
LogicalServers []logicalServer
}
type logicalServer struct {
Name string
ExitCountry string
Region *string
City *string
Servers []physicalServer
}
type physicalServer struct {
EntryIP net.IP
ExitIP net.IP
Domain string
Status uint8
}
func fetchAPI(ctx context.Context, client *http.Client) (
data apiData, err error) {
const url = "https://api.protonmail.ch/vpn/logicals"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return data, err
}
response, err := client.Do(request)
if err != nil {
return data, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return data, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
}
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&data); err != nil {
return data, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
}
if err := response.Body.Close(); err != nil {
return data, err
}
return data, nil
}

14
internal/updater/providers/protonvpn/countries.go Normal file
View File

@@ -0,0 +1,14 @@
package protonvpn
import "strings"
func codeToCountry(countryCode string, countryCodes map[string]string) (
country string, warning string) {
countryCode = strings.ToLower(countryCode)
country, ok := countryCodes[countryCode]
if !ok {
warning = "unknown country code: " + countryCode
country = countryCode
}
return country, warning
}

98
internal/updater/providers/protonvpn/servers.go Normal file
View File

@@ -0,0 +1,98 @@
// Package protonvpn contains code to obtain the server information
// for the ProtonVPN provider.
package protonvpn
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/qdm12/gluetun/internal/constants"
"github.com/qdm12/gluetun/internal/models"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client, minServers int) (
servers []models.ProtonvpnServer, warnings []string, err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, nil, err
}
countryCodes := constants.CountryCodes()
var count int
for _, logicalServer := range data.LogicalServers {
count += len(logicalServer.Servers)
}
if count < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, count, minServers)
}
servers = make([]models.ProtonvpnServer, 0, count)
for _, logicalServer := range data.LogicalServers {
for _, physicalServer := range logicalServer.Servers {
server, warning, err := makeServer(
physicalServer, logicalServer, countryCodes)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
warnings = append(warnings, err.Error())
continue
}
servers = append(servers, server)
}
}
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}
var errServerStatusZero = errors.New("ignoring server with status 0")
func makeServer(physical physicalServer, logical logicalServer,
countryCodes map[string]string) (server models.ProtonvpnServer,
warning string, err error) {
if physical.Status == 0 {
return server, "", fmt.Errorf("%w: %s",
errServerStatusZero, physical.Domain)
}
countryCode := logical.ExitCountry
country, warning := codeToCountry(countryCode, countryCodes)
server = models.ProtonvpnServer{
// Note: for multi-hop use the server name or hostname
// instead of the country
Country: country,
Region: getStringValue(logical.Region),
City: getStringValue(logical.City),
Name: logical.Name,
Hostname: physical.Domain,
EntryIP: physical.EntryIP,
ExitIP: physical.ExitIP,
}
return server, warning, nil
}
func getStringValue(ptr *string) string {
if ptr == nil {
return ""
}
return *ptr
}

26
internal/updater/providers/protonvpn/sort.go Normal file
View File

@@ -0,0 +1,26 @@
package protonvpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.ProtonvpnServer) {
sort.Slice(servers, func(i, j int) bool {
a, b := servers[i], servers[j]
if a.Country == b.Country { //nolint:nestif
if a.Region == b.Region {
if a.City == b.City {
if a.Name == b.Name {
return a.Hostname < b.Hostname
}
return a.Name < b.Name
}
return a.City < b.City
}
return a.Region < b.Region
}
return a.Country < b.Country
})
}

14
internal/updater/providers/protonvpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package protonvpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.ProtonvpnServer) (s string) {
s = "func ProtonvpnServers() []models.ProtonvpnServer {\n"
s += " return []models.ProtonvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

43
internal/updater/providers/purevpn/hosttoserver.go Normal file
View File

@@ -0,0 +1,43 @@
package purevpn
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.PurevpnServer
func (hts hostToServer) add(host string) {
// TODO set TCP and UDP compatibility, set hostname
hts[host] = models.PurevpnServer{}
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.PurevpnServer) {
servers = make([]models.PurevpnServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

33
internal/updater/providers/purevpn/locationtoserver.go Normal file
View File

@@ -0,0 +1,33 @@
package purevpn
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type locationToServer map[string]models.PurevpnServer
func locationKey(country, region, city string) string {
return country + region + city
}
func (lts locationToServer) add(country, region, city string, ips []net.IP) {
key := locationKey(country, region, city)
server, ok := lts[key]
if !ok {
server.Country = country
server.Region = region
server.City = city
}
server.IPs = append(server.IPs, ips...)
lts[key] = server
}
func (lts locationToServer) toServersSlice() (servers []models.PurevpnServer) {
servers = make([]models.PurevpnServer, 0, len(lts))
for _, server := range lts {
servers = append(servers, server)
}
return servers
}

33
internal/updater/providers/purevpn/resolve.go Normal file
View File

@@ -0,0 +1,33 @@
package purevpn
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

98
internal/updater/providers/purevpn/servers.go Normal file
View File

@@ -0,0 +1,98 @@
// Package purevpn contains code to obtain the server information
// for the PureVPN provider.
package purevpn
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/publicip"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client,
unzipper unzip.Unzipper, presolver resolver.Parallel, minServers int) (
servers []models.PurevpnServer, warnings []string, err error) {
const url = "https://s3-us-west-1.amazonaws.com/heartbleed/windows/New+OVPN+Files.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
hts.add(host)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = hts.toServersSlice()
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
// Dedup by location
lts := make(locationToServer)
for _, server := range servers {
country, region, city, err := publicip.Info(ctx, client, server.IPs[0])
if err != nil {
return nil, warnings, err
}
// TODO split servers by host
lts.add(country, region, city, server.IPs)
}
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
servers = lts.toServersSlice()
sortServers(servers)
return servers, warnings, nil
}

19
internal/updater/providers/purevpn/sort.go Normal file
View File

@@ -0,0 +1,19 @@
package purevpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.PurevpnServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].Region == servers[j].Region {
return servers[i].City < servers[j].City
}
return servers[i].Region < servers[j].Region
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/purevpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package purevpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.PurevpnServer) (s string) {
s = "func PurevpnServers() []models.PurevpnServer {\n"
s += " return []models.PurevpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

53
internal/updater/providers/surfshark/api.go Normal file
View File

@@ -0,0 +1,53 @@
package surfshark
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
)
//nolint:unused
type serverData struct {
Host string `json:"connectionName"`
Country string `json:"country"`
Location string `json:"location"`
}
//nolint:unused,deadcode
func fetchAPI(ctx context.Context, client *http.Client) (
servers []serverData, err error) {
const url = "https://my.surfshark.com/vpn/api/v4/server/clusters"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status)
}
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&servers); err != nil {
return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
}
if err := response.Body.Close(); err != nil {
return nil, err
}
return servers, nil
}

59
internal/updater/providers/surfshark/hosttoserver.go Normal file
View File

@@ -0,0 +1,59 @@
package surfshark
import (
"net"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.SurfsharkServer
func (hts hostToServer) add(host, region string) {
// TODO set TCP and UDP
// TODO set hostname
server, ok := hts[host]
if !ok {
server.Region = region
hts[host] = server
}
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) toSubdomainsSlice() (subdomains []string) {
subdomains = make([]string, 0, len(hts))
const suffix = ".prod.surfshark.com"
for host := range hts {
subdomain := strings.TrimSuffix(host, suffix)
subdomains = append(subdomains, subdomain)
}
return subdomains
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.SurfsharkServer) {
servers = make([]models.SurfsharkServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

200
internal/updater/providers/surfshark/regions.go Normal file
View File

@@ -0,0 +1,200 @@
package surfshark
import (
"errors"
"fmt"
"strings"
)
var (
errSuffixNotFound = errors.New("suffix not found")
errSubdomainNotFound = errors.New("subdomain not found in subdomain to region mapping")
)
func parseHost(host string, subdomainToRegion map[string]string) (
region string, err error) {
const suffix = ".prod.surfshark.com"
if !strings.HasSuffix(host, suffix) {
return "", fmt.Errorf("%w: %s in %s",
errSuffixNotFound, suffix, host)
}
subdomain := strings.TrimSuffix(host, suffix)
region, ok := subdomainToRegion[subdomain]
if !ok {
return "", fmt.Errorf("%w: %s", errSubdomainNotFound, subdomain)
}
return region, nil
}
func subdomainToRegion() (mapping map[string]string) {
return map[string]string{
"ae-dub": "United Arab Emirates",
"al-tia": "Albania",
"at-vie": "Austria",
"au-adl": "Australia Adelaide",
"au-bne": "Australia Brisbane",
"au-mel": "Australia Melbourne",
"au-per": "Australia Perth",
"au-syd": "Australia Sydney",
"au-us": "Australia US",
"az-bak": "Azerbaijan",
"ba-sjj": "Bosnia and Herzegovina",
"be-bru": "Belgium",
"bg-sof": "Bulgaria",
"br-sao": "Brazil",
"ca-mon": "Canada Montreal",
"ca-tor": "Canada Toronto",
"ca-us": "Canada US",
"ca-van": "Canada Vancouver",
"ch-zur": "Switzerland",
"cl-san": "Chile",
"co-bog": "Colombia",
"cr-sjn": "Costa Rica",
"cy-nic": "Cyprus",
"cz-prg": "Czech Republic",
"de-ber": "Germany Berlin",
"de-fra": "Germany Frankfurt am Main",
"de-fra-st001": "Germany Frankfurt am Main st001",
"de-fra-st002": "Germany Frankfurt am Main st002",
"de-fra-st003": "Germany Frankfurt am Main st003",
"de-fra-st004": "Germany Frankfurt am Main st004",
"de-fra-st005": "Germany Frankfurt am Main st005",
"de-muc": "Germany Munich",
"de-nue": "Germany Nuremberg",
"de-sg": "Germany Singapour",
"de-uk": "Germany UK",
"dk-cph": "Denmark",
"ee-tll": "Estonia",
"es-bcn": "Spain Barcelona",
"es-mad": "Spain Madrid",
"es-vlc": "Spain Valencia",
"fi-hel": "Finland",
"fr-bod": "France Bordeaux",
"fr-mrs": "France Marseilles",
"fr-par": "France Paris",
"fr-se": "France Sweden",
"gr-ath": "Greece",
"hk-hkg": "Hong Kong",
"hr-zag": "Croatia",
"hu-bud": "Hungary",
"id-jak": "Indonesia",
"ie-dub": "Ireland",
"il-tlv": "Israel",
"in-chn": "India Chennai",
"in-idr": "India Indore",
"in-mum": "India Mumbai",
"in-uk": "India UK",
"is-rkv": "Iceland",
"it-mil": "Italy Milan",
"it-rom": "Italy Rome",
"jp-tok": "Japan Tokyo",
"jp-tok-st001": "Japan Tokyo st001",
"jp-tok-st002": "Japan Tokyo st002",
"jp-tok-st003": "Japan Tokyo st003",
"jp-tok-st004": "Japan Tokyo st004",
"jp-tok-st005": "Japan Tokyo st005",
"jp-tok-st006": "Japan Tokyo st006",
"jp-tok-st007": "Japan Tokyo st007",
"jp-tok-st008": "Japan Tokyo st008",
"jp-tok-st009": "Japan Tokyo st009",
"jp-tok-st010": "Japan Tokyo st010",
"jp-tok-st011": "Japan Tokyo st011",
"jp-tok-st012": "Japan Tokyo st012",
"jp-tok-st013": "Japan Tokyo st013",
"kr-seo": "Korea",
"kz-ura": "Kazakhstan",
"lu-ste": "Luxembourg",
"lv-rig": "Latvia",
"ly-tip": "Libya",
"md-chi": "Moldova",
"mk-skp": "North Macedonia",
"my-kul": "Malaysia",
"ng-lag": "Nigeria",
"nl-ams": "Netherlands Amsterdam",
"nl-ams-st001": "Netherlands Amsterdam st001",
"nl-us": "Netherlands US",
"no-osl": "Norway",
"nz-akl": "New Zealand",
"ph-mnl": "Philippines",
"pl-gdn": "Poland Gdansk",
"pl-waw": "Poland Warsaw",
"pt-lis": "Portugal Lisbon",
"pt-lou": "Portugal Loule",
"pt-opo": "Portugal Porto",
"py-asu": "Paraguay",
"ro-buc": "Romania",
"rs-beg": "Serbia",
"ru-mos": "Russia Moscow",
"ru-spt": "Russia St. Petersburg",
"se-sto": "Sweden",
"sg-hk": "Singapore Hong Kong",
"sg-nl": "Singapore Netherlands",
"sg-sng": "Singapore",
"sg-in": "Singapore in",
"sg-sng-st001": "Singapore st001",
"sg-sng-st002": "Singapore st002",
"sg-sng-st003": "Singapore st003",
"sg-sng-st004": "Singapore st004",
"sg-sng-mp001": "Singapore mp001",
"si-lju": "Slovenia",
"sk-bts": "Slovekia",
"th-bkk": "Thailand",
"tr-bur": "Turkey",
"tw-tai": "Taiwan",
"ua-iev": "Ukraine",
"uk-de": "UK Germany",
"uk-fr": "UK France",
"uk-gla": "UK Glasgow",
"uk-lon": "UK London",
"uk-lon-mp001": "UK London mp001",
"uk-lon-st001": "UK London st001",
"uk-lon-st002": "UK London st002",
"uk-lon-st003": "UK London st003",
"uk-lon-st004": "UK London st004",
"uk-lon-st005": "UK London st005",
"uk-man": "UK Manchester",
"us-atl": "US Atlanta",
"us-bdn": "US Bend",
"us-bos": "US Boston",
"us-buf": "US Buffalo",
"us-chi": "US Chicago",
"us-clt": "US Charlotte",
"us-dal": "US Dallas",
"us-den": "US Denver",
"us-dtw": "US Gahanna",
"us-hou": "US Houston",
"us-kan": "US Kansas City",
"us-las": "US Las Vegas",
"us-lax": "US Los Angeles",
"us-ltm": "US Latham",
"us-mia": "US Miami",
"us-mnz": "US Maryland",
"us-nl": "US Netherlands",
"us-nyc": "US New York City",
"us-nyc-mp001": "US New York City mp001",
"us-nyc-st001": "US New York City st001",
"us-nyc-st002": "US New York City st002",
"us-nyc-st003": "US New York City st003",
"us-nyc-st004": "US New York City st004",
"us-nyc-st005": "US New York City st005",
"us-orl": "US Orlando",
"us-phx": "US Phoenix",
"us-pt": "US Portugal",
"us-sea": "US Seatle",
"us-sfo": "US San Francisco",
"us-slc": "US Salt Lake City",
"us-stl": "US Saint Louis",
"us-tpa": "US Tampa",
"vn-hcm": "Vietnam",
"za-jnb": "South Africa",
"ar-bua": "Argentina Buenos Aires",
"tr-ist": "Turkey Istanbul",
"mx-mex": "Mexico City Mexico",
"ca-tor-mp001": "Canada Toronto mp001",
"de-fra-mp001": "Germany Frankfurt mp001",
"nl-ams-mp001": "Netherlands Amsterdam mp001",
"us-sfo-mp001": "US San Francisco mp001",
}
}

32
internal/updater/providers/surfshark/resolve.go Normal file
View File

@@ -0,0 +1,32 @@
package surfshark
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

130
internal/updater/providers/surfshark/servers.go Normal file
View File

@@ -0,0 +1,130 @@
// Package surfshark contains code to obtain the server information
// for the Surshark provider.
package surfshark
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
presolver resolver.Parallel, minServers int) (
servers []models.SurfsharkServer, warnings []string, err error) {
const url = "https://my.surfshark.com/vpn/api/v1/server/configurations"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
subdomainToRegion := subdomainToRegion()
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
region, err := parseHost(host, subdomainToRegion)
if err != nil {
// treat error as warning and go to next file
warning := err.Error()
warnings = append(warnings, warning)
continue
}
hts.add(host, region)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = hts.toServersSlice()
// process subdomain entries in mapping that were not in the Zip file
subdomainsDone := hts.toSubdomainsSlice()
for _, subdomainDone := range subdomainsDone {
delete(subdomainToRegion, subdomainDone)
}
remainingServers, newWarnings, err := getRemainingServers(
ctx, subdomainToRegion, presolver)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
servers = append(servers, remainingServers...)
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}
func getRemainingServers(ctx context.Context,
subdomainToRegionLeft map[string]string, presolver resolver.Parallel) (
servers []models.SurfsharkServer, warnings []string, err error) {
hosts := make([]string, 0, len(subdomainToRegionLeft))
const suffix = ".prod.surfshark.com"
for subdomain := range subdomainToRegionLeft {
hosts = append(hosts, subdomain+suffix)
}
const minServers = 0
hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers)
if err != nil {
return nil, warnings, err
}
servers = make([]models.SurfsharkServer, 0, len(hostToIPs))
for host, IPs := range hostToIPs {
region, err := parseHost(host, subdomainToRegionLeft)
if err != nil {
return nil, warnings, err
}
server := models.SurfsharkServer{
Region: region,
IPs: IPs,
}
servers = append(servers, server)
}
return servers, warnings, nil
}

13
internal/updater/providers/surfshark/sort.go Normal file
View File

@@ -0,0 +1,13 @@
package surfshark
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.SurfsharkServer) {
sort.Slice(servers, func(i, j int) bool {
return servers[i].Region < servers[j].Region
})
}

14
internal/updater/providers/surfshark/string.go Normal file
View File

@@ -0,0 +1,14 @@
package surfshark
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.SurfsharkServer) (s string) {
s = "func SurfsharkServers() []models.SurfsharkServer {\n"
s += " return []models.SurfsharkServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

31
internal/updater/providers/torguard/filename.go Normal file
View File

@@ -0,0 +1,31 @@
package torguard
import "strings"
func parseFilename(fileName string) (country, city string) {
const prefix = "TorGuard."
const suffix = ".ovpn"
s := strings.TrimPrefix(fileName, prefix)
s = strings.TrimSuffix(s, suffix)
switch {
case strings.Count(s, ".") == 1 && !strings.HasPrefix(s, "USA"):
parts := strings.Split(s, ".")
country = parts[0]
city = parts[1]
case strings.HasPrefix(s, "USA"):
country = "USA"
s = strings.TrimPrefix(s, "USA-")
s = strings.ReplaceAll(s, "-", " ")
s = strings.ReplaceAll(s, ".", " ")
s = strings.ToLower(s)
s = strings.Title(s)
city = s
default:
country = s
}
return country, city
}

76
internal/updater/providers/torguard/servers.go Normal file
View File

@@ -0,0 +1,76 @@
// Package torguard contains code to obtain the server information
// for the Torguard provider.
package torguard
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper, minServers int) (
servers []models.TorguardServer, warnings []string, err error) {
const url = "https://torguard.net/downloads/OpenVPN-TCP-Linux.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
servers = make([]models.TorguardServer, 0, len(contents))
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
country, city := parseFilename(fileName)
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
ip, warning, err := openvpn.ExtractIP(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
server := models.TorguardServer{
Country: country,
City: city,
Hostname: host,
IP: ip,
}
servers = append(servers, server)
}
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}

19
internal/updater/providers/torguard/sort.go Normal file
View File

@@ -0,0 +1,19 @@
package torguard
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.TorguardServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Country < servers[j].Country
})
}

14
internal/updater/providers/torguard/string.go Normal file
View File

@@ -0,0 +1,14 @@
package torguard
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.TorguardServer) (s string) {
s = "func TorguardServers() []models.TorguardServer {\n"
s += " return []models.TorguardServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

22
internal/updater/providers/vyprvpn/filename.go Normal file
View File

@@ -0,0 +1,22 @@
package vyprvpn
import (
"errors"
"fmt"
"strings"
)
var errNotOvpnExt = errors.New("filename does not have the openvpn file extension")
func parseFilename(fileName string) (
region string, err error,
) {
const suffix = ".ovpn"
if !strings.HasSuffix(fileName, suffix) {
return "", fmt.Errorf("%w: %s", errNotOvpnExt, fileName)
}
region = strings.TrimSuffix(fileName, suffix)
region = strings.ReplaceAll(region, " - ", " ")
return region, nil
}

47
internal/updater/providers/vyprvpn/hosttoserver.go Normal file
View File

@@ -0,0 +1,47 @@
package vyprvpn
import (
"net"
"github.com/qdm12/gluetun/internal/models"
)
type hostToServer map[string]models.VyprvpnServer
func (hts hostToServer) add(host, region string) {
server, ok := hts[host]
// TODO set host
if !ok {
server.Region = region
hts[host] = server
}
}
func (hts hostToServer) toHostsSlice() (hosts []string) {
hosts = make([]string, 0, len(hts))
for host := range hts {
hosts = append(hosts, host)
}
return hosts
}
func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) {
for host, IPs := range hostToIPs {
server := hts[host]
server.IPs = IPs
hts[host] = server
}
for host, server := range hts {
if len(server.IPs) == 0 {
delete(hts, host)
}
}
}
func (hts hostToServer) toServersSlice() (servers []models.VyprvpnServer) {
servers = make([]models.VyprvpnServer, 0, len(hts))
for _, server := range hts {
servers = append(servers, server)
}
return servers
}

30
internal/updater/providers/vyprvpn/resolve.go Normal file
View File

@@ -0,0 +1,30 @@
package vyprvpn
import (
"context"
"net"
"time"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func resolveHosts(ctx context.Context, presolver resolver.Parallel,
hosts []string, minServers int) (hostToIPs map[string][]net.IP,
warnings []string, err error) {
const (
maxFailRatio = 0.1
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
MinFound: minServers,
Repeat: resolver.RepeatSettings{
MaxDuration: time.Second,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
return presolver.Resolve(ctx, hosts, settings)
}

82
internal/updater/providers/vyprvpn/servers.go Normal file
View File

@@ -0,0 +1,82 @@
// Package vyprvpn contains code to obtain the server information
// for the VyprVPN provider.
package vyprvpn
import (
"context"
"errors"
"fmt"
"strings"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/openvpn"
"github.com/qdm12/gluetun/internal/updater/resolver"
"github.com/qdm12/gluetun/internal/updater/unzip"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, unzipper unzip.Unzipper,
presolver resolver.Parallel, minServers int) (
servers []models.VyprvpnServer, warnings []string, err error) {
const url = "https://support.vyprvpn.com/hc/article_attachments/360052617332/Vypr_OpenVPN_20200320.zip"
contents, err := unzipper.FetchAndExtract(ctx, url)
if err != nil {
return nil, nil, err
} else if len(contents) < minServers {
return nil, nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(contents), minServers)
}
hts := make(hostToServer)
for fileName, content := range contents {
if !strings.HasSuffix(fileName, ".ovpn") {
continue // not an OpenVPN file
}
host, warning, err := openvpn.ExtractHost(content)
if warning != "" {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warning := err.Error() + " in " + fileName
warnings = append(warnings, warning)
continue
}
region, err := parseFilename(fileName)
if err != nil {
warnings = append(warnings, err.Error())
continue
}
hts.add(host, region)
}
if len(hts) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(hts), minServers)
}
hosts := hts.toHostsSlice()
hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
hts.adaptWithIPs(hostToIPs)
servers = hts.toServersSlice()
if len(servers) < minServers {
return nil, warnings, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, warnings, nil
}

13
internal/updater/providers/vyprvpn/sort.go Normal file
View File

@@ -0,0 +1,13 @@
package vyprvpn
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.VyprvpnServer) {
sort.Slice(servers, func(i, j int) bool {
return servers[i].Region < servers[j].Region
})
}

14
internal/updater/providers/vyprvpn/string.go Normal file
View File

@@ -0,0 +1,14 @@
package vyprvpn
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.VyprvpnServer) (s string) {
s = "func VyprvpnServers() []models.VyprvpnServer {\n"
s += " return []models.VyprvpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

61
internal/updater/providers/windscribe/api.go Normal file
View File

@@ -0,0 +1,61 @@
package windscribe
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"strconv"
"time"
)
var (
ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK")
ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body")
)
type apiData struct {
Data []regionData `json:"data"`
}
type regionData struct {
Region string `json:"name"`
Groups []groupData `json:"groups"`
}
type groupData struct {
City string `json:"city"`
Nodes []serverData `json:"nodes"`
}
type serverData struct {
Hostname string `json:"hostname"`
OpenvpnIP net.IP `json:"ip2"`
}
func fetchAPI(ctx context.Context, client *http.Client) (
data apiData, err error) {
const baseURL = "https://assets.windscribe.com/serverlist/mob-v2/1/"
cacheBreaker := time.Now().Unix()
url := baseURL + strconv.Itoa(int(cacheBreaker))
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return data, err
}
response, err := client.Do(request)
if err != nil {
return data, err
}
defer response.Body.Close()
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&data); err != nil {
return data, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err)
}
return data, nil
}

47
internal/updater/providers/windscribe/servers.go Normal file
View File

@@ -0,0 +1,47 @@
// Package windscribe contains code to obtain the server information
// for the Windscribe provider.
package windscribe
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/qdm12/gluetun/internal/models"
)
var ErrNotEnoughServers = errors.New("not enough servers found")
func GetServers(ctx context.Context, client *http.Client, minServers int) (
servers []models.WindscribeServer, err error) {
data, err := fetchAPI(ctx, client)
if err != nil {
return nil, err
}
for _, regionData := range data.Data {
region := regionData.Region
for _, group := range regionData.Groups {
city := group.City
for _, node := range group.Nodes {
server := models.WindscribeServer{
Region: region,
City: city,
Hostname: node.Hostname,
IP: node.OpenvpnIP,
}
servers = append(servers, server)
}
}
}
if len(servers) < minServers {
return nil, fmt.Errorf("%w: %d and expected at least %d",
ErrNotEnoughServers, len(servers), minServers)
}
sortServers(servers)
return servers, nil
}

19
internal/updater/providers/windscribe/sort.go Normal file
View File

@@ -0,0 +1,19 @@
package windscribe
import (
"sort"
"github.com/qdm12/gluetun/internal/models"
)
func sortServers(servers []models.WindscribeServer) {
sort.Slice(servers, func(i, j int) bool {
if servers[i].Region == servers[j].Region {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Region < servers[j].Region
})
}

14
internal/updater/providers/windscribe/string.go Normal file
View File

@@ -0,0 +1,14 @@
package windscribe
import "github.com/qdm12/gluetun/internal/models"
func Stringify(servers []models.WindscribeServer) (s string) {
s = "func WindscribeServers() []models.WindscribeServer {\n"
s += " return []models.WindscribeServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

136
internal/updater/purevpn.go
View File

@@ -1,136 +0,0 @@
package updater
import (
"context"
"fmt"
"net/http"
"sort"
"strings"
"time"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/publicip"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updatePurevpn(ctx context.Context) (err error) {
servers, warnings, err := findPurevpnServers(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("PureVPN: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Purevpn servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyPurevpnServers(servers))
}
u.servers.Purevpn.Timestamp = u.timeNow().Unix()
u.servers.Purevpn.Servers = servers
return nil
}
func findPurevpnServers(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.PurevpnServer, warnings []string, err error) {
const zipURL = "https://s3-us-west-1.amazonaws.com/heartbleed/windows/New+OVPN+Files.zip"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
hosts := make([]string, 0, len(contents))
for fileName, content := range contents {
if strings.HasSuffix(fileName, "-tcp.ovpn") {
continue // only parse UDP files
}
host, warning, err := extractHostFromOVPN(content)
if len(warning) > 0 {
warnings = append(warnings, warning)
}
if err != nil {
return nil, warnings, fmt.Errorf("%w in %q", err, fileName)
}
hosts = append(hosts, host)
}
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
uniqueServers := make(map[string]models.PurevpnServer, len(hostToIPs))
for host, IPs := range hostToIPs {
if len(IPs) == 0 {
warning := fmt.Sprintf("no IP address found for host %q", host)
warnings = append(warnings, warning)
continue
}
country, region, city, err := publicip.Info(ctx, client, IPs[0])
if err != nil {
return nil, warnings, err
}
key := country + region + city
server, ok := uniqueServers[key]
if ok {
server.IPs = append(server.IPs, IPs...)
} else {
server = models.PurevpnServer{
Country: country,
Region: region,
City: city,
IPs: IPs,
}
}
uniqueServers[key] = server
}
servers = make([]models.PurevpnServer, len(uniqueServers))
i := 0
for _, server := range uniqueServers {
servers[i] = server
i++
}
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].Region == servers[j].Region {
return servers[i].City < servers[j].City
}
return servers[i].Region < servers[j].Region
}
return servers[i].Country < servers[j].Country
})
return servers, warnings, nil
}
func stringifyPurevpnServers(servers []models.PurevpnServer) (s string) {
s = "func PurevpnServers() []models.PurevpnServer {\n"
s += " return []models.PurevpnServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

417
internal/updater/surfshark.go
View File

@@ -1,417 +0,0 @@
package updater
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"strings"
"time"
"github.com/qdm12/gluetun/internal/models"
"github.com/qdm12/gluetun/internal/updater/resolver"
)
func (u *updater) updateSurfshark(ctx context.Context) (err error) {
servers, warnings, err := findSurfsharkServersFromZip(ctx, u.client, u.presolver)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Surfshark: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Surfshark servers: %w", err)
}
if u.options.Stdout {
u.println(stringifySurfsharkServers(servers))
}
u.servers.Surfshark.Timestamp = u.timeNow().Unix()
u.servers.Surfshark.Servers = servers
return nil
}
//nolint:deadcode,unused
func findSurfsharkServersFromAPI(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.SurfsharkServer, warnings []string, err error) {
const url = "https://my.surfshark.com/vpn/api/v4/server/clusters"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url)
}
decoder := json.NewDecoder(response.Body)
var jsonServers []struct {
Host string `json:"connectionName"`
Country string `json:"country"`
Location string `json:"location"`
}
if err := decoder.Decode(&jsonServers); err != nil {
return nil, nil, err
}
if err := response.Body.Close(); err != nil {
return nil, nil, err
}
hosts := make([]string, len(jsonServers))
for i := range jsonServers {
hosts[i] = jsonServers[i].Host
}
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
},
}
hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings)
if err != nil {
return nil, warnings, err
}
for _, jsonServer := range jsonServers {
host := jsonServer.Host
IPs := hostToIPs[host]
if len(IPs) == 0 {
warning := fmt.Sprintf("no IP address found for host %q", host)
warnings = append(warnings, warning)
continue
}
server := models.SurfsharkServer{
Region: jsonServer.Country + " " + jsonServer.Location,
IPs: IPs,
}
servers = append(servers, server)
}
return servers, warnings, nil
}
func findSurfsharkServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) (
servers []models.SurfsharkServer, warnings []string, err error) {
const zipURL = "https://my.surfshark.com/vpn/api/v1/server/configurations"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
mapping := surfsharkSubdomainToRegion()
hosts := make([]string, 0, len(contents))
for fileName, content := range contents {
if strings.HasSuffix(fileName, "_tcp.ovpn") {
continue // only parse UDP files
}
host, warning, err := extractHostFromOVPN(content)
if len(warning) > 0 {
warnings = append(warnings, warning)
}
if err != nil {
// treat error as warning and go to next file
warnings = append(warnings, err.Error()+" in "+fileName)
continue
}
hosts = append(hosts, host)
}
const (
maxFailRatio = 0.1
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
},
}
hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
for host, IPs := range hostToIPs {
if len(IPs) == 0 {
warning := fmt.Sprintf("no IP address found for host %q", host)
warnings = append(warnings, warning)
continue
}
subdomain := strings.TrimSuffix(host, ".prod.surfshark.com")
region, ok := mapping[subdomain]
if ok {
delete(mapping, subdomain)
} else {
region = strings.TrimSuffix(host, ".prod.surfshark.com")
warning := fmt.Sprintf("subdomain %q not found in Surfshark mapping", subdomain)
warnings = append(warnings, warning)
}
server := models.SurfsharkServer{
Region: region,
IPs: IPs,
}
servers = append(servers, server)
}
// process entries in mapping that were not in zip file
remainingServers, newWarnings, err := getRemainingServers(ctx, mapping, presolver)
warnings = append(warnings, newWarnings...)
if err != nil {
return nil, warnings, err
}
servers = append(servers, remainingServers...)
sort.Slice(servers, func(i, j int) bool {
return servers[i].Region < servers[j].Region
})
return servers, warnings, nil
}
func getRemainingServers(ctx context.Context, mapping map[string]string, presolver resolver.Parallel) (
servers []models.SurfsharkServer, warnings []string, err error) {
hosts := make([]string, 0, len(mapping))
for subdomain := range mapping {
hosts = append(hosts, subdomain+".prod.surfshark.com")
}
const (
maxFailRatio = 0.3
maxDuration = 20 * time.Second
betweenDuration = time.Second
maxNoNew = 2
maxFails = 2
)
settings := resolver.ParallelSettings{
MaxFailRatio: maxFailRatio,
Repeat: resolver.RepeatSettings{
MaxDuration: maxDuration,
BetweenDuration: betweenDuration,
MaxNoNew: maxNoNew,
MaxFails: maxFails,
SortIPs: true,
},
}
hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings)
if err != nil {
return nil, warnings, err
}
servers = make([]models.SurfsharkServer, 0, len(hostToIPs))
for host, IPs := range hostToIPs {
subdomain := strings.TrimSuffix(host, ".prod.surfshark.com")
server := models.SurfsharkServer{
Region: mapping[subdomain],
IPs: IPs,
}
servers = append(servers, server)
}
return servers, warnings, nil
}
func stringifySurfsharkServers(servers []models.SurfsharkServer) (s string) {
s = "func SurfsharkServers() []models.SurfsharkServer {\n"
s += " return []models.SurfsharkServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}
func surfsharkSubdomainToRegion() (mapping map[string]string) {
return map[string]string{
"ae-dub": "United Arab Emirates",
"al-tia": "Albania",
"at-vie": "Austria",
"au-adl": "Australia Adelaide",
"au-bne": "Australia Brisbane",
"au-mel": "Australia Melbourne",
"au-per": "Australia Perth",
"au-syd": "Australia Sydney",
"au-us": "Australia US",
"az-bak": "Azerbaijan",
"ba-sjj": "Bosnia and Herzegovina",
"be-bru": "Belgium",
"bg-sof": "Bulgaria",
"br-sao": "Brazil",
"ca-mon": "Canada Montreal",
"ca-tor": "Canada Toronto",
"ca-us": "Canada US",
"ca-van": "Canada Vancouver",
"ch-zur": "Switzerland",
"cl-san": "Chile",
"co-bog": "Colombia",
"cr-sjn": "Costa Rica",
"cy-nic": "Cyprus",
"cz-prg": "Czech Republic",
"de-ber": "Germany Berlin",
"de-fra": "Germany Frankfurt am Main",
"de-fra-st001": "Germany Frankfurt am Main st001",
"de-fra-st002": "Germany Frankfurt am Main st002",
"de-fra-st003": "Germany Frankfurt am Main st003",
"de-fra-st004": "Germany Frankfurt am Main st004",
"de-fra-st005": "Germany Frankfurt am Main st005",
"de-muc": "Germany Munich",
"de-nue": "Germany Nuremberg",
"de-sg": "Germany Singapour",
"de-uk": "Germany UK",
"dk-cph": "Denmark",
"ee-tll": "Estonia",
"es-bcn": "Spain Barcelona",
"es-mad": "Spain Madrid",
"es-vlc": "Spain Valencia",
"fi-hel": "Finland",
"fr-bod": "France Bordeaux",
"fr-mrs": "France Marseilles",
"fr-par": "France Paris",
"fr-se": "France Sweden",
"gr-ath": "Greece",
"hk-hkg": "Hong Kong",
"hr-zag": "Croatia",
"hu-bud": "Hungary",
"id-jak": "Indonesia",
"ie-dub": "Ireland",
"il-tlv": "Israel",
"in-chn": "India Chennai",
"in-idr": "India Indore",
"in-mum": "India Mumbai",
"in-uk": "India UK",
"is-rkv": "Iceland",
"it-mil": "Italy Milan",
"it-rom": "Italy Rome",
"jp-tok": "Japan Tokyo",
"jp-tok-st001": "Japan Tokyo st001",
"jp-tok-st002": "Japan Tokyo st002",
"jp-tok-st003": "Japan Tokyo st003",
"jp-tok-st004": "Japan Tokyo st004",
"jp-tok-st005": "Japan Tokyo st005",
"jp-tok-st006": "Japan Tokyo st006",
"jp-tok-st007": "Japan Tokyo st007",
"jp-tok-st008": "Japan Tokyo st008",
"jp-tok-st009": "Japan Tokyo st009",
"jp-tok-st010": "Japan Tokyo st010",
"jp-tok-st011": "Japan Tokyo st011",
"jp-tok-st012": "Japan Tokyo st012",
"jp-tok-st013": "Japan Tokyo st013",
"kr-seo": "Korea",
"kz-ura": "Kazakhstan",
"lu-ste": "Luxembourg",
"lv-rig": "Latvia",
"ly-tip": "Libya",
"md-chi": "Moldova",
"mk-skp": "North Macedonia",
"my-kul": "Malaysia",
"ng-lag": "Nigeria",
"nl-ams": "Netherlands Amsterdam",
"nl-ams-st001": "Netherlands Amsterdam st001",
"nl-us": "Netherlands US",
"no-osl": "Norway",
"nz-akl": "New Zealand",
"ph-mnl": "Philippines",
"pl-gdn": "Poland Gdansk",
"pl-waw": "Poland Warsaw",
"pt-lis": "Portugal Lisbon",
"pt-lou": "Portugal Loule",
"pt-opo": "Portugal Porto",
"py-asu": "Paraguay",
"ro-buc": "Romania",
"rs-beg": "Serbia",
"ru-mos": "Russia Moscow",
"ru-spt": "Russia St. Petersburg",
"se-sto": "Sweden",
"sg-hk": "Singapore Hong Kong",
"sg-nl": "Singapore Netherlands",
"sg-sng": "Singapore",
"sg-in": "Singapore in",
"sg-sng-st001": "Singapore st001",
"sg-sng-st002": "Singapore st002",
"sg-sng-st003": "Singapore st003",
"sg-sng-st004": "Singapore st004",
"sg-sng-mp001": "Singapore mp001",
"si-lju": "Slovenia",
"sk-bts": "Slovekia",
"th-bkk": "Thailand",
"tr-bur": "Turkey",
"tw-tai": "Taiwan",
"ua-iev": "Ukraine",
"uk-de": "UK Germany",
"uk-fr": "UK France",
"uk-gla": "UK Glasgow",
"uk-lon": "UK London",
"uk-lon-mp001": "UK London mp001",
"uk-lon-st001": "UK London st001",
"uk-lon-st002": "UK London st002",
"uk-lon-st003": "UK London st003",
"uk-lon-st004": "UK London st004",
"uk-lon-st005": "UK London st005",
"uk-man": "UK Manchester",
"us-atl": "US Atlanta",
"us-bdn": "US Bend",
"us-bos": "US Boston",
"us-buf": "US Buffalo",
"us-chi": "US Chicago",
"us-clt": "US Charlotte",
"us-dal": "US Dallas",
"us-den": "US Denver",
"us-dtw": "US Gahanna",
"us-hou": "US Houston",
"us-kan": "US Kansas City",
"us-las": "US Las Vegas",
"us-lax": "US Los Angeles",
"us-ltm": "US Latham",
"us-mia": "US Miami",
"us-mnz": "US Maryland",
"us-nl": "US Netherlands",
"us-nyc": "US New York City",
"us-nyc-mp001": "US New York City mp001",
"us-nyc-st001": "US New York City st001",
"us-nyc-st002": "US New York City st002",
"us-nyc-st003": "US New York City st003",
"us-nyc-st004": "US New York City st004",
"us-nyc-st005": "US New York City st005",
"us-orl": "US Orlando",
"us-phx": "US Phoenix",
"us-pt": "US Portugal",
"us-sea": "US Seatle",
"us-sfo": "US San Francisco",
"us-slc": "US Salt Lake City",
"us-stl": "US Saint Louis",
"us-tpa": "US Tampa",
"vn-hcm": "Vietnam",
"za-jnb": "South Africa",
"ar-bua": "Argentina Buenos Aires",
"tr-ist": "Turkey Istanbul",
"mx-mex": "Mexico City Mexico",
"ca-tor-mp001": "Canada Toronto mp001",
"de-fra-mp001": "Germany Frankfurt mp001",
"nl-ams-mp001": "Netherlands Amsterdam mp001",
"us-sfo-mp001": "US San Francisco mp001",
}
}

114
internal/updater/torguard.go
View File

@@ -1,114 +0,0 @@
package updater
import (
"context"
"fmt"
"net"
"net/http"
"sort"
"strconv"
"strings"
"github.com/qdm12/gluetun/internal/models"
)
func (u *updater) updateTorguard(ctx context.Context) (err error) {
servers, warnings, err := findTorguardServersFromZip(ctx, u.client)
if u.options.CLI {
for _, warning := range warnings {
u.logger.Warn("Torguard: %s", warning)
}
}
if err != nil {
return fmt.Errorf("cannot update Torguard servers: %w", err)
}
if u.options.Stdout {
u.println(stringifyTorguardServers(servers))
}
u.servers.Torguard.Timestamp = u.timeNow().Unix()
u.servers.Torguard.Servers = servers
return nil
}
func findTorguardServersFromZip(ctx context.Context, client *http.Client) (
servers []models.TorguardServer, warnings []string, err error) {
// Note: all servers do both TCP and UDP
const zipURL = "https://torguard.net/downloads/OpenVPN-TCP-Linux.zip"
contents, err := fetchAndExtractFiles(ctx, client, zipURL)
if err != nil {
return nil, nil, err
}
for fileName, content := range contents {
var server models.TorguardServer
const prefix = "TorGuard."
const suffix = ".ovpn"
s := strings.TrimPrefix(fileName, prefix)
s = strings.TrimSuffix(s, suffix)
switch {
case strings.Count(s, ".") == 1 && !strings.HasPrefix(s, "USA"):
parts := strings.Split(s, ".")
server.Country = parts[0]
server.City = parts[1]
case strings.HasPrefix(s, "USA"):
server.Country = "USA"
s = strings.TrimPrefix(s, "USA-")
s = strings.ReplaceAll(s, "-", " ")
s = strings.ReplaceAll(s, ".", " ")
s = strings.ToLower(s)
s = strings.Title(s)
server.City = s
default:
server.Country = s
}
hostnames := extractRemoteHostsFromOpenvpn(content, true, false)
if len(hostnames) != 1 {
warning := "found " + strconv.Itoa(len(hostnames)) +
" hostname(s) instead of 1 in " + fileName
warnings = append(warnings, warning)
continue
}
server.Hostname = hostnames[0]
IPs := extractRemoteHostsFromOpenvpn(content, false, true)
if len(IPs) != 1 {
warning := "found " + strconv.Itoa(len(IPs)) +
" IP(s) instead of 1 in " + fileName
warnings = append(warnings, warning)
continue
}
server.IP = net.ParseIP(IPs[0])
if server.IP == nil {
warnings = append(warnings, "IP address "+IPs[0]+" is not valid in file "+fileName)
}
servers = append(servers, server)
}
sort.Slice(servers, func(i, j int) bool {
if servers[i].Country == servers[j].Country {
if servers[i].City == servers[j].City {
return servers[i].Hostname < servers[j].Hostname
}
return servers[i].City < servers[j].City
}
return servers[i].Country < servers[j].Country
})
return servers, warnings, nil
}
func stringifyTorguardServers(servers []models.TorguardServer) (s string) {
s = "func TorguardServers() []models.TorguardServer {\n"
s += " return []models.TorguardServer{\n"
for _, server := range servers {
s += " " + server.String() + ",\n"
}
s += " }\n"
s += "}"
return s
}

Some files were not shown because too many files have changed in this diff Show More