diff --git a/internal/updater/errors.go b/internal/updater/errors.go deleted file mode 100644 index a716dec0..00000000 --- a/internal/updater/errors.go +++ /dev/null @@ -1,9 +0,0 @@ -package updater - -import "errors" - -var ( - ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") - ErrUnmarshalResponseBody = errors.New("cannot unmarshal response body") - ErrUpdateServerInformation = errors.New("failed updating server information") -) diff --git a/internal/updater/fastestvpn.go b/internal/updater/fastestvpn.go deleted file mode 100644 index 3f344165..00000000 --- a/internal/updater/fastestvpn.go +++ /dev/null @@ -1,161 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net/http" - "regexp" - "sort" - "strings" - "time" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updateFastestvpn(ctx context.Context) (err error) { - servers, warnings, err := findFastestvpnServersFromZip(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("FastestVPN: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update FastestVPN servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyFastestVPNServers(servers)) - } - u.servers.Fastestvpn.Timestamp = u.timeNow().Unix() - u.servers.Fastestvpn.Servers = servers - return nil -} - -func findFastestvpnServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.FastestvpnServer, warnings []string, err error) { - const zipURL = "https://support.fastestvpn.com/download/openvpn-tcp-udp-config-files" - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - trailNumberExp := regexp.MustCompile(`[0-9]+$`) - - type Data struct { - TCP bool - UDP bool - Country string - } - hostToData := make(map[string]Data) - - for fileName, content := range contents { - const ( - tcpSuffix = "-TCP.ovpn" - udpSuffix = "-UDP.ovpn" - ) - var tcp, udp bool - var suffix string - switch { - case strings.HasSuffix(fileName, tcpSuffix): - suffix = tcpSuffix - tcp = true - case strings.HasSuffix(fileName, udpSuffix): - suffix = udpSuffix - udp = true - default: - warning := `filename "` + fileName + `" does not have a protocol suffix` - warnings = append(warnings, warning) - continue - } - - countryWithNumber := strings.TrimSuffix(fileName, suffix) - number := trailNumberExp.FindString(countryWithNumber) - country := countryWithNumber[:len(countryWithNumber)-len(number)] - - host, warning, err := extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - // treat error as warning and go to next file - warnings = append(warnings, err.Error()+" in "+fileName) - continue - } - - data := hostToData[host] - data.Country = country - if tcp { - data.TCP = true - } - if udp { - data.UDP = true - } - hostToData[host] = data - } - - hosts := make([]string, len(hostToData)) - i := 0 - for host := range hostToData { - hosts[i] = host - i++ - } - - const ( - maxFailRatio = 0.1 - maxNoNew = 1 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: time.Second, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - for host, IPs := range hostToIPs { - if len(IPs) == 0 { - warning := fmt.Sprintf("no IP address found for host %q", host) - warnings = append(warnings, warning) - continue - } - - data := hostToData[host] - - server := models.FastestvpnServer{ - Hostname: host, - TCP: data.TCP, - UDP: data.UDP, - Country: data.Country, - IPs: IPs, - } - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - if servers[i].Country == servers[j].Country { - return servers[i].Hostname < servers[j].Hostname - } - return servers[i].Country < servers[j].Country - }) - - return servers, warnings, nil -} - -func stringifyFastestVPNServers(servers []models.FastestvpnServer) (s string) { - s = "func FastestvpnServers() []models.FastestvpnServer {\n" - s += " return []models.FastestvpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/hma.go b/internal/updater/hma.go deleted file mode 100644 index 9461db64..00000000 --- a/internal/updater/hma.go +++ /dev/null @@ -1,288 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "io" - "net/http" - "regexp" - "sort" - "strings" - "time" - "unicode" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updateHideMyAss(ctx context.Context) (err error) { - servers, warnings, err := findHideMyAssServers(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("HideMyAss: %s", warning) - } - } - if err != nil { - return fmt.Errorf("%w: HideMyAss: %s", ErrUpdateServerInformation, err) - } - if u.options.Stdout { - u.println(stringifyHideMyAssServers(servers)) - } - u.servers.HideMyAss.Timestamp = u.timeNow().Unix() - u.servers.HideMyAss.Servers = servers - return nil -} - -func findHideMyAssServers(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.HideMyAssServer, warnings []string, err error) { - TCPhostToURL, err := findHideMyAssHostToURLForProto(ctx, client, "TCP") - if err != nil { - return nil, nil, err - } - - UDPhostToURL, err := findHideMyAssHostToURLForProto(ctx, client, "UDP") - if err != nil { - return nil, nil, err - } - - uniqueHosts := make(map[string]struct{}, len(TCPhostToURL)) - for host := range TCPhostToURL { - uniqueHosts[host] = struct{}{} - } - for host := range UDPhostToURL { - uniqueHosts[host] = struct{}{} - } - - hosts := make([]string, len(uniqueHosts)) - i := 0 - for host := range uniqueHosts { - hosts[i] = host - i++ - } - - const ( - maxFailRatio = 0.1 - maxDuration = 15 * time.Second - betweenDuration = 2 * time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings) - if err != nil { - return nil, warnings, err - } - - servers = make([]models.HideMyAssServer, 0, len(hostToIPs)) - for host, IPs := range hostToIPs { - tcpURL, tcp := TCPhostToURL[host] - udpURL, udp := UDPhostToURL[host] - - var url, protocol string - if tcp { - url = tcpURL - protocol = "TCP" - } else if udp { - url = udpURL - protocol = "UDP" - } - country, region, city := parseHideMyAssURL(url, protocol) - - server := models.HideMyAssServer{ - Country: country, - Region: region, - City: city, - Hostname: host, - IPs: IPs, - TCP: tcp, - UDP: udp, - } - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - return servers[i].Country+servers[i].Region+servers[i].City+servers[i].Hostname < - servers[j].Country+servers[j].Region+servers[j].City+servers[j].Hostname - }) - - return servers, warnings, nil -} - -func findHideMyAssHostToURLForProto(ctx context.Context, client *http.Client, protocol string) ( - hostToURL map[string]string, err error) { - indexURL := "https://vpn.hidemyass.com/vpn-config/" + strings.ToUpper(protocol) + "/" - - urls, err := fetchHideMyAssHTTPIndex(ctx, client, indexURL) - if err != nil { - return nil, err - } - - return fetchMultiOvpnFiles(ctx, client, urls) -} - -func parseHideMyAssURL(url, protocol string) (country, region, city string) { - lastSlashIndex := strings.LastIndex(url, "/") - url = url[lastSlashIndex+1:] - - suffix := "." + strings.ToUpper(protocol) + ".ovpn" - url = strings.TrimSuffix(url, suffix) - - parts := strings.Split(url, ".") - - switch len(parts) { - case 1: - country = parts[0] - return country, "", "" - case 2: //nolint:gomnd - country = parts[0] - city = parts[1] - default: - country = parts[0] - region = parts[1] - city = parts[2] - } - - return camelCaseToWords(country), camelCaseToWords(region), camelCaseToWords(city) -} - -func camelCaseToWords(camelCase string) (words string) { - wasLowerCase := false - for _, r := range camelCase { - if wasLowerCase && unicode.IsUpper(r) { - words += " " - } - wasLowerCase = unicode.IsLower(r) - words += string(r) - } - return words -} - -var hideMyAssIndexRegex = regexp.MustCompile(`.+\.ovpn`) - -func fetchHideMyAssHTTPIndex(ctx context.Context, client *http.Client, indexURL string) (urls []string, err error) { - htmlCode, err := fetchFile(ctx, client, indexURL) - if err != nil { - return nil, err - } - - if !strings.HasSuffix(indexURL, "/") { - indexURL += "/" - } - - lines := strings.Split(string(htmlCode), "\n") - for _, line := range lines { - found := hideMyAssIndexRegex.FindString(line) - if len(found) == 0 { - continue - } - const prefix = `.ovpn">` - const suffix = `` - startIndex := strings.Index(found, prefix) + len(prefix) - endIndex := strings.Index(found, suffix) - filename := found[startIndex:endIndex] - url := indexURL + filename - if !strings.HasSuffix(url, ".ovpn") { - continue - } - urls = append(urls, url) - } - - return urls, nil -} - -func fetchMultiOvpnFiles(ctx context.Context, client *http.Client, urls []string) ( - hostToURL map[string]string, err error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - hostToURL = make(map[string]string, len(urls)) - - type Result struct { - url string - host string - } - results := make(chan Result) - errors := make(chan error) - for _, url := range urls { - go func(url string) { - host, err := fetchOvpnFile(ctx, client, url) - if err != nil { - errors <- fmt.Errorf("%w: for %s", err, url) - return - } - results <- Result{ - url: url, - host: host, - } - }(url) - } - - for range urls { - select { - case newErr := <-errors: - if err == nil { // only assign to the first error - err = newErr - cancel() // stop other operations, this will trigger other errors we ignore - } - case result := <-results: - hostToURL[result.host] = result.url - } - } - - if err != nil { - return nil, err - } - - return hostToURL, nil -} - -func fetchOvpnFile(ctx context.Context, client *http.Client, url string) (hostname string, err error) { - b, err := fetchFile(ctx, client, url) - if err != nil { - return "", err - } - - const rejectIP = true - const rejectDomain = false - hosts := extractRemoteHostsFromOpenvpn(b, rejectIP, rejectDomain) - if len(hosts) == 0 { - return "", errRemoteHostNotFound - } - - return hosts[0], nil -} - -func fetchFile(ctx context.Context, client *http.Client, url string) (b []byte, err error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - - return io.ReadAll(response.Body) -} - -func stringifyHideMyAssServers(servers []models.HideMyAssServer) (s string) { - s = "func HideMyAssServers() []models.HideMyAssServer {\n" - s += " return []models.HideMyAssServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/mullvad.go b/internal/updater/mullvad.go deleted file mode 100644 index 77dfb408..00000000 --- a/internal/updater/mullvad.go +++ /dev/null @@ -1,112 +0,0 @@ -package updater - -import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" - "sort" - "strings" - - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updateMullvad(ctx context.Context) (err error) { - servers, err := findMullvadServers(ctx, u.client) - if err != nil { - return fmt.Errorf("cannot update Mullvad servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyMullvadServers(servers)) - } - u.servers.Mullvad.Timestamp = u.timeNow().Unix() - u.servers.Mullvad.Servers = servers - return nil -} - -func findMullvadServers(ctx context.Context, client *http.Client) (servers []models.MullvadServer, err error) { - const url = "https://api.mullvad.net/www/relays/openvpn/" - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - decoder := json.NewDecoder(response.Body) - var data []struct { - Country string `json:"country_name"` - City string `json:"city_name"` - Active bool `json:"active"` - Owned bool `json:"owned"` - Provider string `json:"provider"` - IPv4 string `json:"ipv4_addr_in"` - IPv6 string `json:"ipv6_addr_in"` - } - if err := decoder.Decode(&data); err != nil { - return nil, err - } - - if err := response.Body.Close(); err != nil { - return nil, err - } - - serversByKey := map[string]models.MullvadServer{} - for _, jsonServer := range data { - if !jsonServer.Active { - continue - } - ipv4 := net.ParseIP(jsonServer.IPv4) - ipv6 := net.ParseIP(jsonServer.IPv6) - if ipv4 == nil || ipv4.To4() == nil { - return nil, fmt.Errorf("cannot parse ipv4 address %q", jsonServer.IPv4) - } else if ipv6 == nil || ipv6.To4() != nil { - return nil, fmt.Errorf("cannot parse ipv6 address %q", jsonServer.IPv6) - } - key := fmt.Sprintf("%s%s%t%s", jsonServer.Country, jsonServer.City, jsonServer.Owned, jsonServer.Provider) - if server, ok := serversByKey[key]; ok { - server.IPs = append(server.IPs, ipv4) - server.IPsV6 = append(server.IPsV6, ipv6) - serversByKey[key] = server - } else { - serversByKey[key] = models.MullvadServer{ - IPs: []net.IP{ipv4}, - IPsV6: []net.IP{ipv6}, - Country: jsonServer.Country, - City: strings.ReplaceAll(jsonServer.City, ",", ""), - ISP: jsonServer.Provider, - Owned: jsonServer.Owned, - } - } - } - for _, server := range serversByKey { - server.IPs = uniqueSortedIPs(server.IPs) - server.IPsV6 = uniqueSortedIPs(server.IPsV6) - servers = append(servers, server) - } - sort.Slice(servers, func(i, j int) bool { - return servers[i].Country+servers[i].City+servers[i].ISP < servers[j].Country+servers[j].City+servers[j].ISP - }) - return servers, nil -} - -func stringifyMullvadServers(servers []models.MullvadServer) (s string) { - s = "func MullvadServers() []models.MullvadServer {\n" - s += " return []models.MullvadServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/nordvpn.go b/internal/updater/nordvpn.go deleted file mode 100644 index b82e8c81..00000000 --- a/internal/updater/nordvpn.go +++ /dev/null @@ -1,125 +0,0 @@ -package updater - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net" - "net/http" - "sort" - "strconv" - "strings" - - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updateNordvpn(ctx context.Context) (err error) { - servers, warnings, err := findNordvpnServers(ctx, u.client) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Nordvpn: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Nordvpn servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyNordvpnServers(servers)) - } - u.servers.Nordvpn.Timestamp = u.timeNow().Unix() - u.servers.Nordvpn.Servers = servers - return nil -} - -var ( - ErrNoIDInServerName = errors.New("no ID in server name") - ErrInvalidIDInServerName = errors.New("invalid ID in server name") -) - -func findNordvpnServers(ctx context.Context, client *http.Client) ( - servers []models.NordvpnServer, warnings []string, err error) { - const url = "https://nordvpn.com/api/server" - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - decoder := json.NewDecoder(response.Body) - var data []struct { - IPAddress string `json:"ip_address"` - Name string `json:"name"` - Country string `json:"country"` - Features struct { - UDP bool `json:"openvpn_udp"` - TCP bool `json:"openvpn_tcp"` - } `json:"features"` - } - if err := decoder.Decode(&data); err != nil { - return nil, nil, err - } - - if err := response.Body.Close(); err != nil { - return nil, nil, err - } - - sort.Slice(data, func(i, j int) bool { - if data[i].Country == data[j].Country { - return data[i].Name < data[j].Name - } - return data[i].Country < data[j].Country - }) - - for _, jsonServer := range data { - if !jsonServer.Features.TCP && !jsonServer.Features.UDP { - warnings = append(warnings, fmt.Sprintf("server %q does not support TCP and UDP for openvpn", jsonServer.Name)) - continue - } - ip := net.ParseIP(jsonServer.IPAddress) - if ip == nil || ip.To4() == nil { - return nil, nil, - fmt.Errorf("IP address %q is not a valid IPv4 address for server %q", - jsonServer.IPAddress, jsonServer.Name) - } - i := strings.IndexRune(jsonServer.Name, '#') - if i < 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrNoIDInServerName, jsonServer.Name) - } - idString := jsonServer.Name[i+1:] - idUint64, err := strconv.ParseUint(idString, 10, 16) - if err != nil { - return nil, nil, fmt.Errorf("%w: %s", ErrInvalidIDInServerName, jsonServer.Name) - } - server := models.NordvpnServer{ - Region: jsonServer.Country, - Number: uint16(idUint64), - IP: ip, - TCP: jsonServer.Features.TCP, - UDP: jsonServer.Features.UDP, - } - servers = append(servers, server) - } - return servers, warnings, nil -} - -func stringifyNordvpnServers(servers []models.NordvpnServer) (s string) { - s = "func NordvpnServers() []models.NordvpnServer {\n" - s += " return []models.NordvpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/openvpn.go b/internal/updater/openvpn.go deleted file mode 100644 index 4f5ec803..00000000 --- a/internal/updater/openvpn.go +++ /dev/null @@ -1,50 +0,0 @@ -package updater - -import ( - "errors" - "fmt" - "net" - "strings" -) - -var ( - errRemoteHostNotFound = errors.New("remote host not found") -) - -func extractHostFromOVPN(b []byte) (host, warning string, err error) { - const ( - rejectIP = true - rejectDomain = false - ) - hosts := extractRemoteHostsFromOpenvpn(b, rejectIP, rejectDomain) - if len(hosts) == 0 { - return "", "", errRemoteHostNotFound - } else if len(hosts) > 1 { - warning = fmt.Sprintf( - "only using the first host %q and discarding %d other hosts", - hosts[0], len(hosts)-1) - } - return hosts[0], warning, nil -} - -func extractRemoteHostsFromOpenvpn(content []byte, - rejectIP, rejectDomain bool) (hosts []string) { - lines := strings.Split(string(content), "\n") - for _, line := range lines { - if !strings.HasPrefix(line, "remote ") { - continue - } - fields := strings.Fields(line) - if len(fields) == 1 || len(fields[1]) == 0 { - continue - } - host := fields[1] - parsedIP := net.ParseIP(host) - if (rejectIP && parsedIP != nil) || - (rejectDomain && parsedIP == nil) { - continue - } - hosts = append(hosts, host) - } - return hosts -} diff --git a/internal/updater/openvpn/extract.go b/internal/updater/openvpn/extract.go new file mode 100644 index 00000000..ba23e1a0 --- /dev/null +++ b/internal/updater/openvpn/extract.go @@ -0,0 +1,66 @@ +package openvpn + +import ( + "errors" + "fmt" + "net" + "strings" +) + +var ( + ErrNoRemoteHost = errors.New("remote host not found") + ErrNoRemoteIP = errors.New("remote IP not found") +) + +func ExtractHost(b []byte) (host, warning string, err error) { + const ( + rejectIP = true + rejectDomain = false + ) + hosts := extractRemoteHosts(b, rejectIP, rejectDomain) + if len(hosts) == 0 { + return "", "", ErrNoRemoteHost + } else if len(hosts) > 1 { + warning = fmt.Sprintf( + "only using the first host %q and discarding %d other hosts", + hosts[0], len(hosts)-1) + } + return hosts[0], warning, nil +} + +func ExtractIP(b []byte) (ip net.IP, warning string, err error) { + const ( + rejectIP = false + rejectDomain = true + ) + ips := extractRemoteHosts(b, rejectIP, rejectDomain) + if len(ips) == 0 { + return nil, "", ErrNoRemoteIP + } else if len(ips) > 1 { + warning = fmt.Sprintf( + "only using the first IP address %s and discarding %d other hosts", + ips[0], len(ips)-1) + } + return net.ParseIP(ips[0]), warning, nil +} + +func extractRemoteHosts(content []byte, rejectIP, rejectDomain bool) (hosts []string) { + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if !strings.HasPrefix(line, "remote ") { + continue + } + fields := strings.Fields(line) + if len(fields) == 1 || len(fields[1]) == 0 { + continue + } + host := fields[1] + parsedIP := net.ParseIP(host) + if (rejectIP && parsedIP != nil) || + (rejectDomain && parsedIP == nil) { + continue + } + hosts = append(hosts, host) + } + return hosts +} diff --git a/internal/updater/openvpn/fetch.go b/internal/updater/openvpn/fetch.go new file mode 100644 index 00000000..c863001c --- /dev/null +++ b/internal/updater/openvpn/fetch.go @@ -0,0 +1,41 @@ +package openvpn + +import ( + "context" + "fmt" + "io" + "net/http" +) + +func FetchFile(ctx context.Context, client *http.Client, url string) ( + host string, err error) { + b, err := fetchData(ctx, client, url) + if err != nil { + return "", err + } + + const rejectIP = true + const rejectDomain = false + hosts := extractRemoteHosts(b, rejectIP, rejectDomain) + if len(hosts) == 0 { + return "", fmt.Errorf("%w for url %s", ErrNoRemoteHost, url) + } + + return hosts[0], nil +} + +func fetchData(ctx context.Context, client *http.Client, url string) ( + b []byte, err error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + return io.ReadAll(response.Body) +} diff --git a/internal/updater/openvpn/multifetch.go b/internal/updater/openvpn/multifetch.go new file mode 100644 index 00000000..2d6ccede --- /dev/null +++ b/internal/updater/openvpn/multifetch.go @@ -0,0 +1,59 @@ +package openvpn + +import ( + "context" + "net/http" +) + +// FetchMultiFiles fetches multiple Openvpn files in parallel and +// parses them to extract each of their host. A mapping from host to +// URL is returned. +func FetchMultiFiles(ctx context.Context, client *http.Client, urls []string) ( + hostToURL map[string]string, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + hostToURL = make(map[string]string, len(urls)) + + type Result struct { + url string + host string + } + + results := make(chan Result) + defer close(results) + errors := make(chan error) + defer close(errors) + + for _, url := range urls { + go func(url string) { + host, err := FetchFile(ctx, client, url) + if err != nil { + errors <- err + return + } + results <- Result{ + url: url, + host: host, + } + }(url) + } + + for range urls { + select { + case newErr := <-errors: + if err == nil { // only assign to the first error + err = newErr + cancel() // stop other operations, this will trigger other errors we ignore + } + case result := <-results: + hostToURL[result.host] = result.url + } + } + + if err != nil { + return nil, err + } + + return hostToURL, nil +} diff --git a/internal/updater/pia.go b/internal/updater/pia.go deleted file mode 100644 index 278d36df..00000000 --- a/internal/updater/pia.go +++ /dev/null @@ -1,145 +0,0 @@ -package updater - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "sort" - - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updatePIA(ctx context.Context) (err error) { - const url = "https://serverlist.piaservers.net/vpninfo/servers/v5" - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - - response, err := u.client.Do(request) - if err != nil { - return err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - b, err := ioutil.ReadAll(response.Body) - if err != nil { - return err - } - - if err := response.Body.Close(); err != nil { - return err - } - - // remove key/signature at the bottom - i := bytes.IndexRune(b, '\n') - b = b[:i] - - var data struct { - Regions []struct { - Name string `json:"name"` - PortForward bool `json:"port_forward"` - Servers struct { - UDP []struct { - IP net.IP `json:"ip"` - CN string `json:"cn"` - } `json:"ovpnudp"` - TCP []struct { - IP net.IP `json:"ip"` - CN string `json:"cn"` - } `json:"ovpntcp"` - } `json:"servers"` - } `json:"regions"` - } - if err := json.Unmarshal(b, &data); err != nil { - return err - } - - // Deduplicate servers with the same IP address - type NetProtocols struct { - tcp, udp bool - } - ipToProtocols := make(map[string]NetProtocols) - - for _, region := range data.Regions { - for _, udpServer := range region.Servers.UDP { - protocols := ipToProtocols[udpServer.IP.String()] - protocols.udp = true - ipToProtocols[udpServer.IP.String()] = protocols - } - for _, tcpServer := range region.Servers.TCP { - protocols := ipToProtocols[tcpServer.IP.String()] - protocols.tcp = true - ipToProtocols[tcpServer.IP.String()] = protocols - } - } - - servers := make([]models.PIAServer, 0, len(ipToProtocols)) // set the capacity, not the length of the slice - for _, region := range data.Regions { - for _, udpServer := range region.Servers.UDP { - protocols, ok := ipToProtocols[udpServer.IP.String()] - if !ok { // already added that IP for a server - continue - } - server := models.PIAServer{ - Region: region.Name, - ServerName: udpServer.CN, - TCP: protocols.tcp, - UDP: protocols.udp, - PortForward: region.PortForward, - IP: udpServer.IP, - } - delete(ipToProtocols, udpServer.IP.String()) - servers = append(servers, server) - } - for _, tcpServer := range region.Servers.TCP { - protocols, ok := ipToProtocols[tcpServer.IP.String()] - if !ok { // already added that IP for a server - continue - } - server := models.PIAServer{ - Region: region.Name, - ServerName: tcpServer.CN, - TCP: protocols.tcp, - UDP: protocols.udp, - PortForward: region.PortForward, - IP: tcpServer.IP, - } - delete(ipToProtocols, tcpServer.IP.String()) - servers = append(servers, server) - } - } - sort.Slice(servers, func(i, j int) bool { - if servers[i].Region == servers[j].Region { - return servers[i].ServerName < servers[j].ServerName - } - return servers[i].Region < servers[j].Region - }) - - if u.options.Stdout { - u.println(stringifyPIAServers(servers)) - } - u.servers.Pia.Timestamp = u.timeNow().Unix() - u.servers.Pia.Servers = servers - return nil -} - -func stringifyPIAServers(servers []models.PIAServer) (s string) { - s = "func PIAServers() []models.PIAServer {\n" - s += " return []models.PIAServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/privado.go b/internal/updater/privado.go deleted file mode 100644 index 5ca3f915..00000000 --- a/internal/updater/privado.go +++ /dev/null @@ -1,106 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net/http" - "sort" - "time" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updatePrivado(ctx context.Context) (err error) { - servers, warnings, err := findPrivadoServersFromZip(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Privado: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Privado servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyPrivadoServers(servers)) - } - u.servers.Privado.Timestamp = u.timeNow().Unix() - u.servers.Privado.Servers = servers - return nil -} - -func findPrivadoServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.PrivadoServer, warnings []string, err error) { - const zipURL = "https://privado.io/apps/ovpn_configs.zip" - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - hosts := make([]string, 0, len(contents)) - for fileName, content := range contents { - hostname, warning, err := extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - return nil, warnings, fmt.Errorf("%w in %q", err, fileName) - } - hosts = append(hosts, hostname) - } - - const ( - maxFailRatio = 0.1 - maxDuration = 3 * time.Second - maxNoNew = 1 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - for hostname, IPs := range hostToIPs { - switch len(IPs) { - case 0: - warning := fmt.Sprintf("no IP address found for host %q", hostname) - warnings = append(warnings, warning) - continue - case 1: - default: - warning := fmt.Sprintf("more than one IP address found for host %q", hostname) - warnings = append(warnings, warning) - } - server := models.PrivadoServer{ - Hostname: hostname, - IP: IPs[0], - } - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - return servers[i].Hostname < servers[j].Hostname - }) - return servers, warnings, nil -} - -func stringifyPrivadoServers(servers []models.PrivadoServer) (s string) { - s = "func PrivadoServers() []models.PrivadoServer {\n" - s += " return []models.PrivadoServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/privatevpn.go b/internal/updater/privatevpn.go deleted file mode 100644 index 4e755c83..00000000 --- a/internal/updater/privatevpn.go +++ /dev/null @@ -1,151 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net/http" - "regexp" - "sort" - "strings" - "time" - - "github.com/qdm12/gluetun/internal/constants" - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updatePrivatevpn(ctx context.Context) (err error) { - servers, warnings, err := findPrivatevpnServersFromZip(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Privatevpn: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Privatevpn servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyPrivatevpnServers(servers)) - } - u.servers.Privatevpn.Timestamp = u.timeNow().Unix() - u.servers.Privatevpn.Servers = servers - return nil -} - -func findPrivatevpnServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.PrivatevpnServer, warnings []string, err error) { - // Note: all servers do both TCP and UDP - const zipURL = "https://privatevpn.com/client/PrivateVPN-TUN.zip" - - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - trailingNumber := regexp.MustCompile(` [0-9]+$`) - countryCodes := constants.CountryCodes() - - uniqueServers := map[string]models.PrivatevpnServer{} // key is the hostname - - for fileName, content := range contents { - const prefix = "PrivateVPN-" - const suffix = "-TUN-443.ovpn" - - if !strings.HasSuffix(fileName, suffix) { - continue // only process TCP servers as they're the same - } - - var server models.PrivatevpnServer - - s := strings.TrimPrefix(fileName, prefix) - s = strings.TrimSuffix(s, suffix) - s = trailingNumber.ReplaceAllString(s, "") - - parts := strings.Split(s, "-") - var countryCode string - countryCode, server.City = parts[0], parts[1] - countryCode = strings.ToLower(countryCode) - var countryCodeOK bool - server.Country, countryCodeOK = countryCodes[countryCode] - if !countryCodeOK { - warnings = append(warnings, "unknown country code: "+countryCode) - server.Country = countryCode - } - - var warning string - server.Hostname, warning, err = extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - return nil, warnings, err - } - if len(warning) > 0 { - continue - } - - uniqueServers[server.Hostname] = server - } - - hostnames := make([]string, len(uniqueServers)) - i := 0 - for hostname := range uniqueServers { - hostnames[i] = hostname - i++ - } - - const ( - maxFailRatio = 0.1 - maxDuration = 6 * time.Second - betweenDuration = time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hostnames, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - for hostname, server := range uniqueServers { - ips := hostToIPs[hostname] - if len(ips) == 0 { - continue - } - server.IPs = ips - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - if servers[i].Country == servers[j].Country { - if servers[i].City == servers[j].City { - return servers[i].Hostname < servers[j].Hostname - } - return servers[i].City < servers[j].City - } - return servers[i].Country < servers[j].Country - }) - - return servers, warnings, nil -} - -func stringifyPrivatevpnServers(servers []models.PrivatevpnServer) (s string) { - s = "func PrivatevpnServers() []models.PrivatevpnServer {\n" - s += " return []models.PrivatevpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/protonvpn.go b/internal/updater/protonvpn.go deleted file mode 100644 index a3120285..00000000 --- a/internal/updater/protonvpn.go +++ /dev/null @@ -1,141 +0,0 @@ -package updater - -import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" - "sort" - "strings" - - "github.com/qdm12/gluetun/internal/constants" - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updateProtonvpn(ctx context.Context) (err error) { - servers, warnings, err := findProtonvpnServers(ctx, u.client) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Protonvpn: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Protonvpn servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyProtonvpnServers(servers)) - } - u.servers.Protonvpn.Timestamp = u.timeNow().Unix() - u.servers.Protonvpn.Servers = servers - return nil -} - -func findProtonvpnServers(ctx context.Context, client *http.Client) ( - servers []models.ProtonvpnServer, warnings []string, err error) { - const url = "https://api.protonmail.ch/vpn/logicals" - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - decoder := json.NewDecoder(response.Body) - var data struct { - LogicalServers []struct { - Name string - ExitCountry string - Region *string - City *string - Servers []struct { - EntryIP net.IP - ExitIP net.IP - Domain string - Status uint8 - } - } - } - if err := decoder.Decode(&data); err != nil { - return nil, nil, err - } - - if err := response.Body.Close(); err != nil { - return nil, nil, err - } - - countryCodesMapping := constants.CountryCodes() - for _, logicalServer := range data.LogicalServers { - for _, physicalServer := range logicalServer.Servers { - if physicalServer.Status == 0 { - warnings = append(warnings, "ignoring server "+physicalServer.Domain+" as its status is 0") - continue - } - - countryCode := strings.ToLower(logicalServer.ExitCountry) - country, ok := countryCodesMapping[countryCode] - if !ok { - warnings = append(warnings, "country not found for country code "+countryCode) - country = logicalServer.ExitCountry - } - - server := models.ProtonvpnServer{ - // Note: for multi-hop use the server name or hostname instead of the country - Country: country, - Region: getStringValue(logicalServer.Region), - City: getStringValue(logicalServer.City), - Name: logicalServer.Name, - Hostname: physicalServer.Domain, - EntryIP: physicalServer.EntryIP, - ExitIP: physicalServer.ExitIP, - } - servers = append(servers, server) - } - } - - sort.Slice(servers, func(i, j int) bool { - a, b := servers[i], servers[j] - if a.Country == b.Country { //nolint:nestif - if a.Region == b.Region { - if a.City == b.City { - if a.Name == b.Name { - return a.Hostname < b.Hostname - } - return a.Name < b.Name - } - return a.City < b.City - } - return a.Region < b.Region - } - return a.Country < b.Country - }) - - return servers, warnings, nil -} - -func getStringValue(ptr *string) string { - if ptr == nil { - return "" - } - return *ptr -} - -func stringifyProtonvpnServers(servers []models.ProtonvpnServer) (s string) { - s = "func ProtonvpnServers() []models.ProtonvpnServer {\n" - s += " return []models.ProtonvpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/providers.go b/internal/updater/providers.go new file mode 100644 index 00000000..dea45c44 --- /dev/null +++ b/internal/updater/providers.go @@ -0,0 +1,279 @@ +package updater + +import ( + "context" + "fmt" + + "github.com/qdm12/gluetun/internal/updater/providers/cyberghost" + "github.com/qdm12/gluetun/internal/updater/providers/fastestvpn" + "github.com/qdm12/gluetun/internal/updater/providers/hidemyass" + "github.com/qdm12/gluetun/internal/updater/providers/mullvad" + "github.com/qdm12/gluetun/internal/updater/providers/nordvpn" + "github.com/qdm12/gluetun/internal/updater/providers/pia" + "github.com/qdm12/gluetun/internal/updater/providers/privado" + "github.com/qdm12/gluetun/internal/updater/providers/privatevpn" + "github.com/qdm12/gluetun/internal/updater/providers/protonvpn" + "github.com/qdm12/gluetun/internal/updater/providers/purevpn" + "github.com/qdm12/gluetun/internal/updater/providers/surfshark" + "github.com/qdm12/gluetun/internal/updater/providers/torguard" + "github.com/qdm12/gluetun/internal/updater/providers/vyprvpn" + "github.com/qdm12/gluetun/internal/updater/providers/windscribe" +) + +func (u *updater) updateCyberghost(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Cyberghost.Servers)) + servers, err := cyberghost.GetServers(ctx, u.presolver, minServers) + if err != nil { + return err + } + if u.options.Stdout { + u.println(cyberghost.Stringify(servers)) + } + u.servers.Cyberghost.Timestamp = u.timeNow().Unix() + u.servers.Cyberghost.Servers = servers + return nil +} + +func (u *updater) updateFastestvpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Fastestvpn.Servers)) + servers, warnings, err := fastestvpn.GetServers( + ctx, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("FastestVPN: " + warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(fastestvpn.Stringify(servers)) + } + u.servers.Fastestvpn.Timestamp = u.timeNow().Unix() + u.servers.Fastestvpn.Servers = servers + return nil +} + +func (u *updater) updateHideMyAss(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.HideMyAss.Servers)) + servers, warnings, err := hidemyass.GetServers( + ctx, u.client, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("HideMyAss: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(hidemyass.Stringify(servers)) + } + u.servers.HideMyAss.Timestamp = u.timeNow().Unix() + u.servers.HideMyAss.Servers = servers + return nil +} + +func (u *updater) updateMullvad(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Mullvad.Servers)) + servers, err := mullvad.GetServers(ctx, u.client, minServers) + if err != nil { + return err + } + if u.options.Stdout { + u.println(mullvad.Stringify(servers)) + } + u.servers.Mullvad.Timestamp = u.timeNow().Unix() + u.servers.Mullvad.Servers = servers + return nil +} + +func (u *updater) updateNordvpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Nordvpn.Servers)) + servers, warnings, err := nordvpn.GetServers(ctx, u.client, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("NordVPN: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(nordvpn.Stringify(servers)) + } + u.servers.Nordvpn.Timestamp = u.timeNow().Unix() + u.servers.Nordvpn.Servers = servers + return nil +} + +func (u *updater) updatePIA(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Pia.Servers)) + servers, err := pia.GetServers(ctx, u.client, minServers) + if err != nil { + return err + } + if u.options.Stdout { + u.println(pia.Stringify(servers)) + } + u.servers.Pia.Timestamp = u.timeNow().Unix() + u.servers.Pia.Servers = servers + return nil +} + +func (u *updater) updatePrivado(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Privado.Servers)) + servers, warnings, err := privado.GetServers( + ctx, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("Privado: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(privado.Stringify(servers)) + } + u.servers.Privado.Timestamp = u.timeNow().Unix() + u.servers.Privado.Servers = servers + return nil +} + +func (u *updater) updatePrivatevpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Privatevpn.Servers)) + servers, warnings, err := privatevpn.GetServers( + ctx, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("PrivateVPN: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(privatevpn.Stringify(servers)) + } + u.servers.Privatevpn.Timestamp = u.timeNow().Unix() + u.servers.Privatevpn.Servers = servers + return nil +} + +func (u *updater) updateProtonvpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Privatevpn.Servers)) + servers, warnings, err := protonvpn.GetServers(ctx, u.client, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("ProtonVPN: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(protonvpn.Stringify(servers)) + } + u.servers.Protonvpn.Timestamp = u.timeNow().Unix() + u.servers.Protonvpn.Servers = servers + return nil +} + +func (u *updater) updatePurevpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Purevpn.Servers)) + servers, warnings, err := purevpn.GetServers( + ctx, u.client, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("PureVPN: %s", warning) + } + } + if err != nil { + return fmt.Errorf("cannot update Purevpn servers: %w", err) + } + if u.options.Stdout { + u.println(purevpn.Stringify(servers)) + } + u.servers.Purevpn.Timestamp = u.timeNow().Unix() + u.servers.Purevpn.Servers = servers + return nil +} + +func (u *updater) updateSurfshark(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Surfshark.Servers)) + servers, warnings, err := surfshark.GetServers( + ctx, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("Surfshark: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(surfshark.Stringify(servers)) + } + u.servers.Surfshark.Timestamp = u.timeNow().Unix() + u.servers.Surfshark.Servers = servers + return nil +} + +func (u *updater) updateTorguard(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Torguard.Servers)) + servers, warnings, err := torguard.GetServers(ctx, u.unzipper, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("Torguard: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(torguard.Stringify(servers)) + } + u.servers.Torguard.Timestamp = u.timeNow().Unix() + u.servers.Torguard.Servers = servers + return nil +} + +func (u *updater) updateVyprvpn(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Vyprvpn.Servers)) + servers, warnings, err := vyprvpn.GetServers( + ctx, u.unzipper, u.presolver, minServers) + if u.options.CLI { + for _, warning := range warnings { + u.logger.Warn("VyprVPN: %s", warning) + } + } + if err != nil { + return err + } + if u.options.Stdout { + u.println(vyprvpn.Stringify(servers)) + } + u.servers.Vyprvpn.Timestamp = u.timeNow().Unix() + u.servers.Vyprvpn.Servers = servers + return nil +} + +func (u *updater) updateWindscribe(ctx context.Context) (err error) { + minServers := getMinServers(len(u.servers.Windscribe.Servers)) + servers, err := windscribe.GetServers(ctx, u.client, minServers) + if err != nil { + return err + } + if u.options.Stdout { + u.println(windscribe.Stringify(servers)) + } + u.servers.Windscribe.Timestamp = u.timeNow().Unix() + u.servers.Windscribe.Servers = servers + return nil +} + +func getMinServers(existingServers int) (minServers int) { + const minRatio = 0.8 + return int(minRatio * float64(existingServers)) +} diff --git a/internal/updater/cyberghost.go b/internal/updater/providers/cyberghost/constants.go similarity index 62% rename from internal/updater/cyberghost.go rename to internal/updater/providers/cyberghost/constants.go index bae91f5b..3472d7c7 100644 --- a/internal/updater/cyberghost.go +++ b/internal/updater/providers/cyberghost/constants.go @@ -1,117 +1,6 @@ -package updater +package cyberghost -import ( - "context" - "sort" - "time" - - "github.com/qdm12/gluetun/internal/constants" - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updateCyberghost(ctx context.Context) (err error) { - servers, err := findCyberghostServers(ctx, u.presolver) - if err != nil { - return err - } - if u.options.Stdout { - u.println(stringifyCyberghostServers(servers)) - } - u.servers.Cyberghost.Timestamp = u.timeNow().Unix() - u.servers.Cyberghost.Servers = servers - return nil -} - -func findCyberghostServers(ctx context.Context, presolver resolver.Parallel) ( - servers []models.CyberghostServer, err error) { - groups := getCyberghostGroups() - allCountryCodes := constants.CountryCodes() - cyberghostCountryCodes := getCyberghostSubdomainToRegion() - possibleCountryCodes := mergeCountryCodes(cyberghostCountryCodes, allCountryCodes) - - // key is the host - possibleServers := make(map[string]models.CyberghostServer, len(groups)*len(possibleCountryCodes)) - possibleHosts := make([]string, 0, len(groups)*len(possibleCountryCodes)) - for groupID, groupName := range groups { - for countryCode, region := range possibleCountryCodes { - const domain = "cg-dialup.net" - possibleHost := groupID + "-" + countryCode + "." + domain - possibleHosts = append(possibleHosts, possibleHost) - possibleServer := models.CyberghostServer{ - Region: region, - Group: groupName, - } - possibleServers[possibleHost] = possibleServer - } - } - - const ( - maxFailRatio = 1 - minFound = 100 - maxDuration = 10 * time.Second - maxNoNew = 2 - maxFails = 1 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - MinFound: minFound, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: time.Second, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, _, err := presolver.Resolve(ctx, possibleHosts, settings) - if err != nil { - return nil, err - } - - if err := ctx.Err(); err != nil { - return nil, err - } - - // Set IPs for servers found - for host, IPs := range hostToIPs { - server := possibleServers[host] - server.IPs = IPs - possibleServers[host] = server - } - - // Remove servers with no IPs (aka not found) - for host, server := range possibleServers { - if len(server.IPs) == 0 { - delete(possibleServers, host) - } - } - - // Flatten possibleServers to a slice - servers = make([]models.CyberghostServer, 0, len(possibleServers)) - for _, server := range possibleServers { - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - return servers[i].Region < servers[j].Region - }) - return servers, nil -} - -//nolint:goconst -func stringifyCyberghostServers(servers []models.CyberghostServer) (s string) { - s = "func CyberghostServers() []models.CyberghostServer {\n" - s += " return []models.CyberghostServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} - -func getCyberghostGroups() map[string]string { +func getGroups() map[string]string { return map[string]string{ "87-1": "Premium UDP Europe", "94-1": "Premium UDP USA", @@ -124,21 +13,7 @@ func getCyberghostGroups() map[string]string { } } -func mergeCountryCodes(base, extend map[string]string) (merged map[string]string) { - merged = make(map[string]string, len(base)) - for countryCode, region := range base { - merged[countryCode] = region - } - for countryCode := range base { - delete(extend, countryCode) - } - for countryCode, region := range extend { - merged[countryCode] = region - } - return merged -} - -func getCyberghostSubdomainToRegion() map[string]string { +func getSubdomainToRegion() map[string]string { return map[string]string{ "af": "Afghanistan", "ax": "Aland Islands", diff --git a/internal/updater/providers/cyberghost/countries.go b/internal/updater/providers/cyberghost/countries.go new file mode 100644 index 00000000..4a770201 --- /dev/null +++ b/internal/updater/providers/cyberghost/countries.go @@ -0,0 +1,15 @@ +package cyberghost + +func mergeCountryCodes(base, extend map[string]string) (merged map[string]string) { + merged = make(map[string]string, len(base)) + for countryCode, region := range base { + merged[countryCode] = region + } + for countryCode := range base { + delete(extend, countryCode) + } + for countryCode, region := range extend { + merged[countryCode] = region + } + return merged +} diff --git a/internal/updater/providers/cyberghost/hosttoserver.go b/internal/updater/providers/cyberghost/hosttoserver.go new file mode 100644 index 00000000..812e813c --- /dev/null +++ b/internal/updater/providers/cyberghost/hosttoserver.go @@ -0,0 +1,65 @@ +package cyberghost + +import ( + "net" + + "github.com/qdm12/gluetun/internal/constants" + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.CyberghostServer + +func getPossibleServers() (possibleServers hostToServer) { + groups := getGroups() + + cyberghostCountryCodes := getSubdomainToRegion() + allCountryCodes := constants.CountryCodes() + possibleCountryCodes := mergeCountryCodes(cyberghostCountryCodes, allCountryCodes) + + n := len(groups) * len(possibleCountryCodes) + + possibleServers = make(hostToServer, n) // key is the host + + for groupID, groupName := range groups { + for countryCode, region := range possibleCountryCodes { + const domain = "cg-dialup.net" + possibleHost := groupID + "-" + countryCode + "." + domain + possibleServer := models.CyberghostServer{ + Region: region, + Group: groupName, + } + possibleServers[possibleHost] = possibleServer + } + } + + return possibleServers +} + +func (hts hostToServer) hostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toSlice() (servers []models.CyberghostServer) { + servers = make([]models.CyberghostServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/cyberghost/resolve.go b/internal/updater/providers/cyberghost/resolve.go new file mode 100644 index 00000000..786adca1 --- /dev/null +++ b/internal/updater/providers/cyberghost/resolve.go @@ -0,0 +1,42 @@ +package cyberghost + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + possibleHosts []string, minServers int) ( + hostToIPs map[string][]net.IP, err error) { + const ( + maxFailRatio = 1 + maxDuration = 10 * time.Second + betweenDuration = 500 * time.Millisecond + maxNoNew = 2 + maxFails = 10 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + BetweenDuration: betweenDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + hostToIPs, _, err = presolver.Resolve(ctx, possibleHosts, settings) + if err != nil { + return nil, err + } + + if err := ctx.Err(); err != nil { + return nil, err + } + + return hostToIPs, nil +} diff --git a/internal/updater/providers/cyberghost/servers.go b/internal/updater/providers/cyberghost/servers.go new file mode 100644 index 00000000..2c912b9f --- /dev/null +++ b/internal/updater/providers/cyberghost/servers.go @@ -0,0 +1,28 @@ +// Package cyberghost contains code to obtain the server information +// for the Cyberghost provider. +package cyberghost + +import ( + "context" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func GetServers(ctx context.Context, presolver resolver.Parallel, + minServers int) (servers []models.CyberghostServer, err error) { + possibleServers := getPossibleServers() + + possibleHosts := possibleServers.hostsSlice() + hostToIPs, err := resolveHosts(ctx, presolver, possibleHosts, minServers) + if err != nil { + return nil, err + } + + possibleServers.adaptWithIPs(hostToIPs) + + servers = possibleServers.toSlice() + + sortServers(servers) + return servers, nil +} diff --git a/internal/updater/providers/cyberghost/sort.go b/internal/updater/providers/cyberghost/sort.go new file mode 100644 index 00000000..1926335c --- /dev/null +++ b/internal/updater/providers/cyberghost/sort.go @@ -0,0 +1,16 @@ +package cyberghost + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.CyberghostServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Region == servers[j].Region { + return servers[i].Group < servers[j].Group + } + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/cyberghost/string.go b/internal/updater/providers/cyberghost/string.go new file mode 100644 index 00000000..94c779fa --- /dev/null +++ b/internal/updater/providers/cyberghost/string.go @@ -0,0 +1,15 @@ +package cyberghost + +import "github.com/qdm12/gluetun/internal/models" + +// Stringify converts servers to code string format. +func Stringify(servers []models.CyberghostServer) (s string) { + s = "func CyberghostServers() []models.CyberghostServer {\n" + s += " return []models.CyberghostServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/fastestvpn/filename.go b/internal/updater/providers/fastestvpn/filename.go new file mode 100644 index 00000000..26abb31f --- /dev/null +++ b/internal/updater/providers/fastestvpn/filename.go @@ -0,0 +1,39 @@ +package fastestvpn + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var errFilenameNoProtocolSuffix = errors.New("filename does not have a protocol suffix") + +var trailNumberExp = regexp.MustCompile(`[0-9]+$`) + +func parseFilename(fileName string) ( + country string, tcp, udp bool, err error, +) { + const ( + tcpSuffix = "-TCP.ovpn" + udpSuffix = "-UDP.ovpn" + ) + var suffix string + switch { + case strings.HasSuffix(fileName, tcpSuffix): + suffix = tcpSuffix + tcp = true + case strings.HasSuffix(fileName, udpSuffix): + suffix = udpSuffix + udp = true + default: + return "", false, false, fmt.Errorf("%w: %s", + errFilenameNoProtocolSuffix, fileName) + } + + countryWithNumber := strings.TrimSuffix(fileName, suffix) + number := trailNumberExp.FindString(countryWithNumber) + country = countryWithNumber[:len(countryWithNumber)-len(number)] + + return country, tcp, udp, nil +} diff --git a/internal/updater/providers/fastestvpn/hosttoserver.go b/internal/updater/providers/fastestvpn/hosttoserver.go new file mode 100644 index 00000000..c5d686f2 --- /dev/null +++ b/internal/updater/providers/fastestvpn/hosttoserver.go @@ -0,0 +1,53 @@ +package fastestvpn + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.FastestvpnServer + +func (hts hostToServer) add(host, country string, tcp, udp bool) { + server, ok := hts[host] + if !ok { + server.Hostname = host + server.Country = country + } + if tcp { + server.TCP = true + } + if udp { + server.UDP = true + } + hts[host] = server +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toServersSlice() (servers []models.FastestvpnServer) { + servers = make([]models.FastestvpnServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/fastestvpn/resolve.go b/internal/updater/providers/fastestvpn/resolve.go new file mode 100644 index 00000000..93c1bb10 --- /dev/null +++ b/internal/updater/providers/fastestvpn/resolve.go @@ -0,0 +1,30 @@ +package fastestvpn + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxNoNew = 1 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: time.Second, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/fastestvpn/servers.go b/internal/updater/providers/fastestvpn/servers.go new file mode 100644 index 00000000..30ca0a5b --- /dev/null +++ b/internal/updater/providers/fastestvpn/servers.go @@ -0,0 +1,82 @@ +// Package fastestvpn contains code to obtain the server information +// for the FastestVPN provider. +package fastestvpn + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, + presolver resolver.Parallel, minServers int) ( + servers []models.FastestvpnServer, warnings []string, err error) { + const url = "https://support.fastestvpn.com/download/openvpn-tcp-udp-config-files" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + country, tcp, udp, err := parseFilename(fileName) + if err != nil { + warnings = append(warnings, err.Error()) + continue + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + hts.add(host, country, tcp, udp) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + hts.adaptWithIPs(hostToIPs) + + servers = hts.toServersSlice() + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/fastestvpn/sort.go b/internal/updater/providers/fastestvpn/sort.go new file mode 100644 index 00000000..e5fb642c --- /dev/null +++ b/internal/updater/providers/fastestvpn/sort.go @@ -0,0 +1,16 @@ +package fastestvpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.FastestvpnServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + return servers[i].Hostname < servers[j].Hostname + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/fastestvpn/string.go b/internal/updater/providers/fastestvpn/string.go new file mode 100644 index 00000000..f0017ea4 --- /dev/null +++ b/internal/updater/providers/fastestvpn/string.go @@ -0,0 +1,14 @@ +package fastestvpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.FastestvpnServer) (s string) { + s = "func FastestvpnServers() []models.FastestvpnServer {\n" + s += " return []models.FastestvpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/hidemyass/hosts.go b/internal/updater/providers/hidemyass/hosts.go new file mode 100644 index 00000000..07856f8e --- /dev/null +++ b/internal/updater/providers/hidemyass/hosts.go @@ -0,0 +1,19 @@ +package hidemyass + +func getUniqueHosts(tcpHostToURL, udpHostToURL map[string]string) ( + hosts []string) { + uniqueHosts := make(map[string]struct{}, len(tcpHostToURL)) + for host := range tcpHostToURL { + uniqueHosts[host] = struct{}{} + } + for host := range udpHostToURL { + uniqueHosts[host] = struct{}{} + } + + hosts = make([]string, 0, len(uniqueHosts)) + for host := range uniqueHosts { + hosts = append(hosts, host) + } + + return hosts +} diff --git a/internal/updater/providers/hidemyass/hosttourl.go b/internal/updater/providers/hidemyass/hosttourl.go new file mode 100644 index 00000000..0b10440f --- /dev/null +++ b/internal/updater/providers/hidemyass/hosttourl.go @@ -0,0 +1,37 @@ +package hidemyass + +import ( + "context" + "net/http" + "strings" + + "github.com/qdm12/gluetun/internal/updater/openvpn" +) + +func getAllHostToURL(ctx context.Context, client *http.Client) ( + tcpHostToURL, udpHostToURL map[string]string, err error) { + tcpHostToURL, err = getHostToURL(ctx, client, "TCP") + if err != nil { + return nil, nil, err + } + + udpHostToURL, err = getHostToURL(ctx, client, "UDP") + if err != nil { + return nil, nil, err + } + + return tcpHostToURL, udpHostToURL, nil +} + +func getHostToURL(ctx context.Context, client *http.Client, protocol string) ( + hostToURL map[string]string, err error) { + const baseURL = "https://vpn.hidemyass.com/vpn-config" + indexURL := baseURL + "/" + strings.ToUpper(protocol) + "/" + + urls, err := fetchIndex(ctx, client, indexURL) + if err != nil { + return nil, err + } + + return openvpn.FetchMultiFiles(ctx, client, urls) +} diff --git a/internal/updater/providers/hidemyass/index.go b/internal/updater/providers/hidemyass/index.go new file mode 100644 index 00000000..efa998d1 --- /dev/null +++ b/internal/updater/providers/hidemyass/index.go @@ -0,0 +1,54 @@ +package hidemyass + +import ( + "context" + "io" + "net/http" + "regexp" + "strings" +) + +var indexOpenvpnLinksRegex = regexp.MustCompile(`.+\.ovpn`) + +func fetchIndex(ctx context.Context, client *http.Client, indexURL string) ( + openvpnURLs []string, err error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + htmlCode, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if !strings.HasSuffix(indexURL, "/") { + indexURL += "/" + } + + lines := strings.Split(string(htmlCode), "\n") + for _, line := range lines { + found := indexOpenvpnLinksRegex.FindString(line) + if len(found) == 0 { + continue + } + const prefix = `.ovpn">` + const suffix = `` + startIndex := strings.Index(found, prefix) + len(prefix) + endIndex := strings.Index(found, suffix) + filename := found[startIndex:endIndex] + openvpnURL := indexURL + filename + if !strings.HasSuffix(openvpnURL, ".ovpn") { + continue + } + openvpnURLs = append(openvpnURLs, openvpnURL) + } + + return openvpnURLs, nil +} diff --git a/internal/updater/providers/hidemyass/resolve.go b/internal/updater/providers/hidemyass/resolve.go new file mode 100644 index 00000000..171e7228 --- /dev/null +++ b/internal/updater/providers/hidemyass/resolve.go @@ -0,0 +1,33 @@ +package hidemyass + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) ( + hostToIPs map[string][]net.IP, warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxDuration = 15 * time.Second + betweenDuration = 2 * time.Second + maxNoNew = 2 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + BetweenDuration: betweenDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/hidemyass/servers.go b/internal/updater/providers/hidemyass/servers.go new file mode 100644 index 00000000..c7710834 --- /dev/null +++ b/internal/updater/providers/hidemyass/servers.go @@ -0,0 +1,68 @@ +// Package hidemyass contains code to obtain the server information +// for the HideMyAss provider. +package hidemyass + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, + presolver resolver.Parallel, minServers int) ( + servers []models.HideMyAssServer, warnings []string, err error) { + tcpHostToURL, udpHostToURL, err := getAllHostToURL(ctx, client) + if err != nil { + return nil, nil, err + } + + hosts := getUniqueHosts(tcpHostToURL, udpHostToURL) + + if len(hosts) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hosts), minServers) + } + + hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers) + if err != nil { + return nil, warnings, err + } + + servers = make([]models.HideMyAssServer, 0, len(hostToIPs)) + for host, IPs := range hostToIPs { + tcpURL, tcp := tcpHostToURL[host] + udpURL, udp := udpHostToURL[host] + + // These two are only used to extract the country, region and city. + var url, protocol string + if tcp { + url = tcpURL + protocol = "TCP" + } else if udp { + url = udpURL + protocol = "UDP" + } + country, region, city := parseOpenvpnURL(url, protocol) + + server := models.HideMyAssServer{ + Country: country, + Region: region, + City: city, + Hostname: host, + IPs: IPs, + TCP: tcp, + UDP: udp, + } + servers = append(servers, server) + } + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/hidemyass/sort.go b/internal/updater/providers/hidemyass/sort.go new file mode 100644 index 00000000..edbdc35a --- /dev/null +++ b/internal/updater/providers/hidemyass/sort.go @@ -0,0 +1,22 @@ +package hidemyass + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.HideMyAssServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + if servers[i].Region == servers[j].Region { + if servers[i].City == servers[j].City { + return servers[i].Hostname < servers[j].Hostname + } + return servers[i].City < servers[j].City + } + return servers[i].Region < servers[j].Region + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/hidemyass/string.go b/internal/updater/providers/hidemyass/string.go new file mode 100644 index 00000000..b159c67b --- /dev/null +++ b/internal/updater/providers/hidemyass/string.go @@ -0,0 +1,14 @@ +package hidemyass + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.HideMyAssServer) (s string) { + s = "func HideMyAssServers() []models.HideMyAssServer {\n" + s += " return []models.HideMyAssServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/hidemyass/url.go b/internal/updater/providers/hidemyass/url.go new file mode 100644 index 00000000..44aadde8 --- /dev/null +++ b/internal/updater/providers/hidemyass/url.go @@ -0,0 +1,44 @@ +package hidemyass + +import ( + "strings" + "unicode" +) + +func parseOpenvpnURL(url, protocol string) (country, region, city string) { + lastSlashIndex := strings.LastIndex(url, "/") + url = url[lastSlashIndex+1:] + + suffix := "." + strings.ToUpper(protocol) + ".ovpn" + url = strings.TrimSuffix(url, suffix) + + parts := strings.Split(url, ".") + + switch len(parts) { + case 1: + country = parts[0] + return country, "", "" + case 2: //nolint:gomnd + country = parts[0] + city = parts[1] + default: + country = parts[0] + region = parts[1] + city = parts[2] + } + + return camelCaseToWords(country), camelCaseToWords(region), + camelCaseToWords(city) +} + +func camelCaseToWords(camelCase string) (words string) { + wasLowerCase := false + for _, r := range camelCase { + if wasLowerCase && unicode.IsUpper(r) { + words += " " + } + wasLowerCase = unicode.IsLower(r) + words += string(r) + } + return words +} diff --git a/internal/updater/providers/mullvad/api.go b/internal/updater/providers/mullvad/api.go new file mode 100644 index 00000000..a527fbbc --- /dev/null +++ b/internal/updater/providers/mullvad/api.go @@ -0,0 +1,55 @@ +package mullvad + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") + ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body") +) + +type serverData struct { + Hostname string `json:"hostname"` + Country string `json:"country_name"` + City string `json:"city_name"` + Active bool `json:"active"` + Owned bool `json:"owned"` + Provider string `json:"provider"` + IPv4 string `json:"ipv4_addr_in"` + IPv6 string `json:"ipv6_addr_in"` +} + +func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) { + const url = "https://api.mullvad.net/www/relays/openvpn/" + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) + } + + decoder := json.NewDecoder(response.Body) + if err := decoder.Decode(&data); err != nil { + return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) + } + + if err := response.Body.Close(); err != nil { + return nil, err + } + + return data, nil +} diff --git a/internal/updater/providers/mullvad/hosttoserver.go b/internal/updater/providers/mullvad/hosttoserver.go new file mode 100644 index 00000000..5018b216 --- /dev/null +++ b/internal/updater/providers/mullvad/hosttoserver.go @@ -0,0 +1,58 @@ +package mullvad + +import ( + "errors" + "fmt" + "net" + "strings" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.MullvadServer + +var ( + ErrParseIPv4 = errors.New("cannot parse IPv4 address") + ErrParseIPv6 = errors.New("cannot parse IPv6 address") +) + +func (hts hostToServer) add(data serverData) (err error) { + if !data.Active { + return + } + + ipv4 := net.ParseIP(data.IPv4) + if ipv4 == nil || ipv4.To4() == nil { + return fmt.Errorf("%w: %s", ErrParseIPv4, data.IPv4) + } + + ipv6 := net.ParseIP(data.IPv6) + if ipv6 == nil || ipv6.To4() != nil { + return fmt.Errorf("%w: %s", ErrParseIPv6, data.IPv6) + } + + server, ok := hts[data.Hostname] + if !ok { + server.Country = data.Country + server.City = strings.ReplaceAll(data.City, ",", "") + server.ISP = data.Provider + server.Owned = data.Owned + } + + server.IPs = append(server.IPs, ipv4) + server.IPsV6 = append(server.IPsV6, ipv6) + + hts[data.Hostname] = server + + return nil +} + +func (hts hostToServer) toServersSlice() (servers []models.MullvadServer) { + servers = make([]models.MullvadServer, 0, len(hts)) + for _, server := range hts { + server.IPs = uniqueSortedIPs(server.IPs) + server.IPsV6 = uniqueSortedIPs(server.IPsV6) + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/ips.go b/internal/updater/providers/mullvad/ips.go similarity index 96% rename from internal/updater/ips.go rename to internal/updater/providers/mullvad/ips.go index 85589a69..651b5b4d 100644 --- a/internal/updater/ips.go +++ b/internal/updater/providers/mullvad/ips.go @@ -1,4 +1,4 @@ -package updater +package mullvad import ( "bytes" @@ -12,6 +12,7 @@ func uniqueSortedIPs(ips []net.IP) []net.IP { key := ip.String() uniqueIPs[key] = struct{}{} } + ips = make([]net.IP, 0, len(uniqueIPs)) for key := range uniqueIPs { ip := net.ParseIP(key) @@ -20,8 +21,10 @@ func uniqueSortedIPs(ips []net.IP) []net.IP { } ips = append(ips, ip) } + sort.Slice(ips, func(i, j int) bool { return bytes.Compare(ips[i], ips[j]) < 0 }) + return ips } diff --git a/internal/updater/ips_test.go b/internal/updater/providers/mullvad/ips_test.go similarity index 98% rename from internal/updater/ips_test.go rename to internal/updater/providers/mullvad/ips_test.go index b2ea1051..18a35735 100644 --- a/internal/updater/ips_test.go +++ b/internal/updater/providers/mullvad/ips_test.go @@ -1,4 +1,4 @@ -package updater +package mullvad import ( "net" diff --git a/internal/updater/providers/mullvad/servers.go b/internal/updater/providers/mullvad/servers.go new file mode 100644 index 00000000..240785fb --- /dev/null +++ b/internal/updater/providers/mullvad/servers.go @@ -0,0 +1,68 @@ +// Package mullvad contains code to obtain the server information +// for the Mullvad provider. +package mullvad + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/qdm12/gluetun/internal/models" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, minServers int) ( + servers []models.MullvadServer, err error) { + data, err := fetchAPI(ctx, client) + if err != nil { + return nil, err + } + + hts := make(hostToServer) + for _, serverData := range data { + if err := hts.add(serverData); err != nil { + return nil, err + } + } + + if len(hts) < minServers { + return nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + servers = hts.toServersSlice() + + servers = groupByProperties(servers) + + sortServers(servers) + + return servers, nil +} + +// TODO group by hostname so remove this. +func groupByProperties(serversByHost []models.MullvadServer) (serversByProps []models.MullvadServer) { + propsToServer := make(map[string]models.MullvadServer, len(serversByHost)) + for _, server := range serversByHost { + key := server.Country + server.City + server.ISP + strconv.FormatBool(server.Owned) + serverByProps, ok := propsToServer[key] + if !ok { + serverByProps.Country = server.Country + serverByProps.City = server.City + serverByProps.ISP = server.ISP + serverByProps.Owned = server.Owned + } + serverByProps.IPs = append(serverByProps.IPs, server.IPs...) + serverByProps.IPsV6 = append(serverByProps.IPsV6, server.IPsV6...) + propsToServer[key] = serverByProps + } + + serversByProps = make([]models.MullvadServer, 0, len(propsToServer)) + for _, serverByProp := range propsToServer { + serversByProps = append(serversByProps, serverByProp) + } + + return serversByProps +} diff --git a/internal/updater/providers/mullvad/sort.go b/internal/updater/providers/mullvad/sort.go new file mode 100644 index 00000000..7a694ce7 --- /dev/null +++ b/internal/updater/providers/mullvad/sort.go @@ -0,0 +1,19 @@ +package mullvad + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.MullvadServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + if servers[i].City == servers[j].City { + return servers[i].ISP < servers[j].ISP + } + return servers[i].City < servers[j].City + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/mullvad/string.go b/internal/updater/providers/mullvad/string.go new file mode 100644 index 00000000..f9d9e2d8 --- /dev/null +++ b/internal/updater/providers/mullvad/string.go @@ -0,0 +1,14 @@ +package mullvad + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.MullvadServer) (s string) { + s = "func MullvadServers() []models.MullvadServer {\n" + s += " return []models.MullvadServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/mullvad_test.go b/internal/updater/providers/mullvad/string_test.go similarity index 86% rename from internal/updater/mullvad_test.go rename to internal/updater/providers/mullvad/string_test.go index b2a305f7..bde33706 100644 --- a/internal/updater/mullvad_test.go +++ b/internal/updater/providers/mullvad/string_test.go @@ -1,4 +1,4 @@ -package updater +package mullvad import ( "net" @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_stringifyMullvadServers(t *testing.T) { +func Test_Stringify(t *testing.T) { servers := []models.MullvadServer{{ Country: "webland", City: "webcity", @@ -27,6 +27,6 @@ func MullvadServers() []models.MullvadServer { } ` expected = strings.TrimPrefix(strings.TrimSuffix(expected, "\n"), "\n") - s := stringifyMullvadServers(servers) + s := Stringify(servers) assert.Equal(t, expected, s) } diff --git a/internal/updater/providers/nordvpn/api.go b/internal/updater/providers/nordvpn/api.go new file mode 100644 index 00000000..b7359100 --- /dev/null +++ b/internal/updater/providers/nordvpn/api.go @@ -0,0 +1,55 @@ +package nordvpn + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") + ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body") +) + +type serverData struct { + Domain string `json:"domain"` + IPAddress string `json:"ip_address"` + Name string `json:"name"` + Country string `json:"country"` + Features struct { + UDP bool `json:"openvpn_udp"` + TCP bool `json:"openvpn_tcp"` + } `json:"features"` +} + +func fetchAPI(ctx context.Context, client *http.Client) (data []serverData, err error) { + const url = "https://nordvpn.com/api/server" + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) + } + + decoder := json.NewDecoder(response.Body) + if err := decoder.Decode(&data); err != nil { + return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) + } + + if err := response.Body.Close(); err != nil { + return nil, err + } + + return data, nil +} diff --git a/internal/updater/providers/nordvpn/ip.go b/internal/updater/providers/nordvpn/ip.go new file mode 100644 index 00000000..fcc6cff1 --- /dev/null +++ b/internal/updater/providers/nordvpn/ip.go @@ -0,0 +1,16 @@ +package nordvpn + +import ( + "fmt" + "net" +) + +func parseIPv4(s string) (ipv4 net.IP, err error) { + ip := net.ParseIP(s) + if ip == nil { + return nil, fmt.Errorf("%w: %q", ErrParseIP, s) + } else if ip.To4() == nil { + return nil, fmt.Errorf("%w: %s", ErrNotIPv4, ip) + } + return ip, nil +} diff --git a/internal/updater/providers/nordvpn/name.go b/internal/updater/providers/nordvpn/name.go new file mode 100644 index 00000000..1fbd1518 --- /dev/null +++ b/internal/updater/providers/nordvpn/name.go @@ -0,0 +1,29 @@ +package nordvpn + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +var ( + ErrNoIDInServerName = errors.New("no ID in server name") + ErrInvalidIDInServerName = errors.New("invalid ID in server name") +) + +func parseServerName(serverName string) (number uint16, err error) { + i := strings.IndexRune(serverName, '#') + if i < 0 { + return 0, fmt.Errorf("%w: %s", ErrNoIDInServerName, serverName) + } + + idString := serverName[i+1:] + idUint64, err := strconv.ParseUint(idString, 10, 16) + if err != nil { + return 0, fmt.Errorf("%w: %s", ErrInvalidIDInServerName, serverName) + } + + number = uint16(idUint64) + return number, nil +} diff --git a/internal/updater/providers/nordvpn/servers.go b/internal/updater/providers/nordvpn/servers.go new file mode 100644 index 00000000..a30e4b27 --- /dev/null +++ b/internal/updater/providers/nordvpn/servers.go @@ -0,0 +1,64 @@ +// Package nordvpn contains code to obtain the server information +// for the NordVPN provider. +package nordvpn + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/qdm12/gluetun/internal/models" +) + +var ( + ErrParseIP = errors.New("cannot parse IP address") + ErrNotIPv4 = errors.New("IP address is not IPv4") + ErrNotEnoughServers = errors.New("not enough servers found") +) + +func GetServers(ctx context.Context, client *http.Client, minServers int) ( + servers []models.NordvpnServer, warnings []string, err error) { + data, err := fetchAPI(ctx, client) + if err != nil { + return nil, nil, err + } + + servers = make([]models.NordvpnServer, 0, len(data)) + + for _, jsonServer := range data { + if !jsonServer.Features.TCP && !jsonServer.Features.UDP { + warning := "server does not support TCP and UDP for openvpn: " + jsonServer.Name + warnings = append(warnings, warning) + continue + } + + ip, err := parseIPv4(jsonServer.IPAddress) + if err != nil { + return nil, nil, fmt.Errorf("%w for server %s", err, jsonServer.Name) + } + + number, err := parseServerName(jsonServer.Name) + if err != nil { + return nil, nil, err + } + + server := models.NordvpnServer{ + Region: jsonServer.Country, + Number: number, + IP: ip, + TCP: jsonServer.Features.TCP, + UDP: jsonServer.Features.UDP, + } + servers = append(servers, server) + } + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/nordvpn/sort.go b/internal/updater/providers/nordvpn/sort.go new file mode 100644 index 00000000..4812a531 --- /dev/null +++ b/internal/updater/providers/nordvpn/sort.go @@ -0,0 +1,16 @@ +package nordvpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.NordvpnServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Region == servers[j].Region { + return servers[i].Number < servers[j].Number + } + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/nordvpn/string.go b/internal/updater/providers/nordvpn/string.go new file mode 100644 index 00000000..b8fbc72b --- /dev/null +++ b/internal/updater/providers/nordvpn/string.go @@ -0,0 +1,14 @@ +package nordvpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.NordvpnServer) (s string) { + s = "func NordvpnServers() []models.NordvpnServer {\n" + s += " return []models.NordvpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/pia/api.go b/internal/updater/providers/pia/api.go new file mode 100644 index 00000000..e230d1e6 --- /dev/null +++ b/internal/updater/providers/pia/api.go @@ -0,0 +1,74 @@ +package pia + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") +) + +type apiData struct { + Regions []regionData `json:"regions"` +} + +type regionData struct { + Name string `json:"name"` + PortForward bool `json:"port_forward"` + Offline bool `json:"offline"` + Servers struct { + UDP []serverData `json:"ovpnudp"` + TCP []serverData `json:"ovpntcp"` + } `json:"servers"` +} + +type serverData struct { + IP net.IP `json:"ip"` + CN string `json:"cn"` +} + +func fetchAPI(ctx context.Context, client *http.Client) ( + data apiData, err error) { + const url = "https://serverlist.piaservers.net/vpninfo/servers/v5" + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return data, err + } + + response, err := client.Do(request) + if err != nil { + return data, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return data, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) + } + + b, err := ioutil.ReadAll(response.Body) + if err != nil { + return data, err + } + + if err := response.Body.Close(); err != nil { + return data, err + } + + // remove key/signature at the bottom + i := bytes.IndexRune(b, '\n') + b = b[:i] + + if err := json.Unmarshal(b, &data); err != nil { + return data, err + } + + return data, nil +} diff --git a/internal/updater/providers/pia/servers.go b/internal/updater/providers/pia/servers.go new file mode 100644 index 00000000..22618301 --- /dev/null +++ b/internal/updater/providers/pia/servers.go @@ -0,0 +1,90 @@ +// Package pia contains code to obtain the server information +// for the Private Internet Access provider. +package pia + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/qdm12/gluetun/internal/models" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, minServers int) ( + servers []models.PIAServer, err error) { + data, err := fetchAPI(ctx, client) + if err != nil { + return nil, err + } + + for _, region := range data.Regions { + // Deduplicate servers with the same common name + commonNameToProtocols := dedupByProtocol(region) + + // newServers can support only UDP or both TCP and UDP + newServers := dataToServers(region.Servers.UDP, region.Name, + region.PortForward, commonNameToProtocols) + servers = append(servers, newServers...) + + // tcpServers only support TCP as mixed servers were found above. + tcpServers := dataToServers(region.Servers.TCP, region.Name, + region.PortForward, commonNameToProtocols) + servers = append(servers, tcpServers...) + } + + if len(servers) < minServers { + return nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, nil +} + +type protocols struct { + tcp bool + udp bool +} + +// Deduplicate servers with the same common name for different protocols. +func dedupByProtocol(region regionData) (commonNameToProtocols map[string]protocols) { + commonNameToProtocols = make(map[string]protocols) + for _, udpServer := range region.Servers.UDP { + protocols := commonNameToProtocols[udpServer.CN] + protocols.udp = true + commonNameToProtocols[udpServer.CN] = protocols + } + for _, tcpServer := range region.Servers.TCP { + protocols := commonNameToProtocols[tcpServer.CN] + protocols.tcp = true + commonNameToProtocols[tcpServer.CN] = protocols + } + return commonNameToProtocols +} + +func dataToServers(data []serverData, region string, portForward bool, + commonNameToProtocols map[string]protocols) ( + servers []models.PIAServer) { + servers = make([]models.PIAServer, 0, len(data)) + for _, serverData := range data { + proto, ok := commonNameToProtocols[serverData.CN] + if !ok { + continue // server already added + } + delete(commonNameToProtocols, serverData.CN) + server := models.PIAServer{ + Region: region, + ServerName: serverData.CN, + TCP: proto.tcp, + UDP: proto.udp, + PortForward: portForward, + IP: serverData.IP, + } + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/pia/sort.go b/internal/updater/providers/pia/sort.go new file mode 100644 index 00000000..4c3ed095 --- /dev/null +++ b/internal/updater/providers/pia/sort.go @@ -0,0 +1,16 @@ +package pia + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.PIAServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Region == servers[j].Region { + return servers[i].ServerName < servers[j].ServerName + } + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/pia/string.go b/internal/updater/providers/pia/string.go new file mode 100644 index 00000000..0353d54f --- /dev/null +++ b/internal/updater/providers/pia/string.go @@ -0,0 +1,14 @@ +package pia + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.PIAServer) (s string) { + s = "func PIAServers() []models.PIAServer {\n" + s += " return []models.PIAServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/privado/hosttoserver.go b/internal/updater/providers/privado/hosttoserver.go new file mode 100644 index 00000000..609db3f6 --- /dev/null +++ b/internal/updater/providers/privado/hosttoserver.go @@ -0,0 +1,53 @@ +package privado + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.PrivadoServer + +func (hts hostToServer) add(host string) { + server, ok := hts[host] + if ok { + return + } + server.Hostname = host + hts[host] = server +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) ( + warnings []string) { + for host, IPs := range hostToIPs { + if len(IPs) > 1 { + warning := "more than one IP address found for host " + host + warnings = append(warnings, warning) + } + server := hts[host] + server.IP = IPs[0] + hts[host] = server + } + for host, server := range hts { + if server.IP == nil { + delete(hts, host) + } + } + return warnings +} + +func (hts hostToServer) toServersSlice() (servers []models.PrivadoServer) { + servers = make([]models.PrivadoServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/privado/resolve.go b/internal/updater/providers/privado/resolve.go new file mode 100644 index 00000000..90a780c5 --- /dev/null +++ b/internal/updater/providers/privado/resolve.go @@ -0,0 +1,31 @@ +package privado + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxDuration = 3 * time.Second + maxNoNew = 1 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/privado/servers.go b/internal/updater/providers/privado/servers.go new file mode 100644 index 00000000..1297b655 --- /dev/null +++ b/internal/updater/providers/privado/servers.go @@ -0,0 +1,72 @@ +// Package privado contains code to obtain the server information +// for the Privado provider. +package privado + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, + presolver resolver.Parallel, minServers int) ( + servers []models.PrivadoServer, warnings []string, err error) { + const url = "https://privado.io/apps/ovpn_configs.zip" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + hts.add(host) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + newWarnings = hts.adaptWithIPs(hostToIPs) + warnings = append(warnings, newWarnings...) + + servers = hts.toServersSlice() + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/privado/sort.go b/internal/updater/providers/privado/sort.go new file mode 100644 index 00000000..22e108d0 --- /dev/null +++ b/internal/updater/providers/privado/sort.go @@ -0,0 +1,13 @@ +package privado + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.PrivadoServer) { + sort.Slice(servers, func(i, j int) bool { + return servers[i].Hostname < servers[j].Hostname + }) +} diff --git a/internal/updater/providers/privado/string.go b/internal/updater/providers/privado/string.go new file mode 100644 index 00000000..e848cf30 --- /dev/null +++ b/internal/updater/providers/privado/string.go @@ -0,0 +1,14 @@ +package privado + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.PrivadoServer) (s string) { + s = "func PrivadoServers() []models.PrivadoServer {\n" + s += " return []models.PrivadoServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/privatevpn/countries.go b/internal/updater/providers/privatevpn/countries.go new file mode 100644 index 00000000..ba9f6b3f --- /dev/null +++ b/internal/updater/providers/privatevpn/countries.go @@ -0,0 +1,14 @@ +package privatevpn + +import "strings" + +func codeToCountry(countryCode string, countryCodes map[string]string) ( + country string, warning string) { + countryCode = strings.ToLower(countryCode) + country, ok := countryCodes[countryCode] + if !ok { + warning = "unknown country code: " + countryCode + country = countryCode + } + return country, warning +} diff --git a/internal/updater/providers/privatevpn/filename.go b/internal/updater/providers/privatevpn/filename.go new file mode 100644 index 00000000..bbd64eef --- /dev/null +++ b/internal/updater/providers/privatevpn/filename.go @@ -0,0 +1,54 @@ +package privatevpn + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + trailingNumber = regexp.MustCompile(` [0-9]+$`) +) + +var ( + errBadPrefix = errors.New("bad prefix in file name") + errBadSuffix = errors.New("bad suffix in file name") + errNotEnoughParts = errors.New("not enough parts in file name") +) + +func parseFilename(fileName string) ( + countryCode, city string, err error, +) { + fileName = strings.ReplaceAll(fileName, " ", "") // remove spaces + + const prefix = "PrivateVPN-" + if !strings.HasPrefix(fileName, prefix) { + return "", "", fmt.Errorf("%w: %s", errBadPrefix, fileName) + } + s := strings.TrimPrefix(fileName, prefix) + + const tcpSuffix = "-TUN-443.ovpn" + const udpSuffix = "-TUN-1194.ovpn" + switch { + case strings.HasSuffix(fileName, tcpSuffix): + s = strings.TrimSuffix(s, tcpSuffix) + case strings.HasSuffix(fileName, udpSuffix): + s = strings.TrimSuffix(s, udpSuffix) + default: + return "", "", fmt.Errorf("%w: %s", errBadSuffix, fileName) + } + + s = trailingNumber.ReplaceAllString(s, "") + + parts := strings.Split(s, "-") + const minParts = 2 + if len(parts) < minParts { + return "", "", fmt.Errorf("%w: %s", + errNotEnoughParts, fileName) + } + countryCode, city = parts[0], parts[1] + countryCode = strings.ToLower(countryCode) + + return countryCode, city, nil +} diff --git a/internal/updater/providers/privatevpn/hosttoserver.go b/internal/updater/providers/privatevpn/hosttoserver.go new file mode 100644 index 00000000..42aa0f86 --- /dev/null +++ b/internal/updater/providers/privatevpn/hosttoserver.go @@ -0,0 +1,50 @@ +package privatevpn + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.PrivatevpnServer + +// TODO check if server supports TCP and UDP. +func (hts hostToServer) add(host, country, city string) { + server, ok := hts[host] + if ok { + return + } + server.Hostname = host + server.Country = country + server.City = city + hts[host] = server +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toServersSlice() (servers []models.PrivatevpnServer) { + servers = make([]models.PrivatevpnServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/privatevpn/resolve.go b/internal/updater/providers/privatevpn/resolve.go new file mode 100644 index 00000000..f80fa9b3 --- /dev/null +++ b/internal/updater/providers/privatevpn/resolve.go @@ -0,0 +1,33 @@ +package privatevpn + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxDuration = 6 * time.Second + betweenDuration = time.Second + maxNoNew = 2 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + BetweenDuration: betweenDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/privatevpn/servers.go b/internal/updater/providers/privatevpn/servers.go new file mode 100644 index 00000000..031ed4e0 --- /dev/null +++ b/internal/updater/providers/privatevpn/servers.go @@ -0,0 +1,86 @@ +// Package privatevpn contains code to obtain the server information +// for the PrivateVPN provider. +package privatevpn + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/constants" + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, + presolver resolver.Parallel, minServers int) ( + servers []models.PrivatevpnServer, warnings []string, err error) { + const url = "https://privatevpn.com/client/PrivateVPN-TUN.zip" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + countryCodes := constants.CountryCodes() + + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + countryCode, city, err := parseFilename(fileName) + if err != nil { + warnings = append(warnings, err.Error()) + continue + } + + country, warning := codeToCountry(countryCode, countryCodes) + if warning != "" { + warnings = append(warnings, warning) + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + hts.add(host, country, city) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + hts.adaptWithIPs(hostToIPs) + + servers = hts.toServersSlice() + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/privatevpn/sort.go b/internal/updater/providers/privatevpn/sort.go new file mode 100644 index 00000000..b3742a17 --- /dev/null +++ b/internal/updater/providers/privatevpn/sort.go @@ -0,0 +1,19 @@ +package privatevpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.PrivatevpnServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + if servers[i].City == servers[j].City { + return servers[i].Hostname < servers[j].Hostname + } + return servers[i].City < servers[j].City + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/privatevpn/string.go b/internal/updater/providers/privatevpn/string.go new file mode 100644 index 00000000..65f70b15 --- /dev/null +++ b/internal/updater/providers/privatevpn/string.go @@ -0,0 +1,14 @@ +package privatevpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.PrivatevpnServer) (s string) { + s = "func PrivatevpnServers() []models.PrivatevpnServer {\n" + s += " return []models.PrivatevpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/protonvpn/api.go b/internal/updater/providers/protonvpn/api.go new file mode 100644 index 00000000..3cc5ad23 --- /dev/null +++ b/internal/updater/providers/protonvpn/api.go @@ -0,0 +1,65 @@ +package protonvpn + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") + ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body") +) + +type apiData struct { + LogicalServers []logicalServer +} + +type logicalServer struct { + Name string + ExitCountry string + Region *string + City *string + Servers []physicalServer +} + +type physicalServer struct { + EntryIP net.IP + ExitIP net.IP + Domain string + Status uint8 +} + +func fetchAPI(ctx context.Context, client *http.Client) ( + data apiData, err error) { + const url = "https://api.protonmail.ch/vpn/logicals" + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return data, err + } + + response, err := client.Do(request) + if err != nil { + return data, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return data, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) + } + + decoder := json.NewDecoder(response.Body) + if err := decoder.Decode(&data); err != nil { + return data, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) + } + + if err := response.Body.Close(); err != nil { + return data, err + } + + return data, nil +} diff --git a/internal/updater/providers/protonvpn/countries.go b/internal/updater/providers/protonvpn/countries.go new file mode 100644 index 00000000..accc0c90 --- /dev/null +++ b/internal/updater/providers/protonvpn/countries.go @@ -0,0 +1,14 @@ +package protonvpn + +import "strings" + +func codeToCountry(countryCode string, countryCodes map[string]string) ( + country string, warning string) { + countryCode = strings.ToLower(countryCode) + country, ok := countryCodes[countryCode] + if !ok { + warning = "unknown country code: " + countryCode + country = countryCode + } + return country, warning +} diff --git a/internal/updater/providers/protonvpn/servers.go b/internal/updater/providers/protonvpn/servers.go new file mode 100644 index 00000000..0f7482ea --- /dev/null +++ b/internal/updater/providers/protonvpn/servers.go @@ -0,0 +1,98 @@ +// Package protonvpn contains code to obtain the server information +// for the ProtonVPN provider. +package protonvpn + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/qdm12/gluetun/internal/constants" + "github.com/qdm12/gluetun/internal/models" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, minServers int) ( + servers []models.ProtonvpnServer, warnings []string, err error) { + data, err := fetchAPI(ctx, client) + if err != nil { + return nil, nil, err + } + + countryCodes := constants.CountryCodes() + + var count int + for _, logicalServer := range data.LogicalServers { + count += len(logicalServer.Servers) + } + + if count < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, count, minServers) + } + + servers = make([]models.ProtonvpnServer, 0, count) + for _, logicalServer := range data.LogicalServers { + for _, physicalServer := range logicalServer.Servers { + server, warning, err := makeServer( + physicalServer, logicalServer, countryCodes) + + if warning != "" { + warnings = append(warnings, warning) + } + + if err != nil { + warnings = append(warnings, err.Error()) + continue + } + + servers = append(servers, server) + } + } + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} + +var errServerStatusZero = errors.New("ignoring server with status 0") + +func makeServer(physical physicalServer, logical logicalServer, + countryCodes map[string]string) (server models.ProtonvpnServer, + warning string, err error) { + if physical.Status == 0 { + return server, "", fmt.Errorf("%w: %s", + errServerStatusZero, physical.Domain) + } + + countryCode := logical.ExitCountry + country, warning := codeToCountry(countryCode, countryCodes) + + server = models.ProtonvpnServer{ + // Note: for multi-hop use the server name or hostname + // instead of the country + Country: country, + Region: getStringValue(logical.Region), + City: getStringValue(logical.City), + Name: logical.Name, + Hostname: physical.Domain, + EntryIP: physical.EntryIP, + ExitIP: physical.ExitIP, + } + + return server, warning, nil +} + +func getStringValue(ptr *string) string { + if ptr == nil { + return "" + } + return *ptr +} diff --git a/internal/updater/providers/protonvpn/sort.go b/internal/updater/providers/protonvpn/sort.go new file mode 100644 index 00000000..4336911f --- /dev/null +++ b/internal/updater/providers/protonvpn/sort.go @@ -0,0 +1,26 @@ +package protonvpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.ProtonvpnServer) { + sort.Slice(servers, func(i, j int) bool { + a, b := servers[i], servers[j] + if a.Country == b.Country { //nolint:nestif + if a.Region == b.Region { + if a.City == b.City { + if a.Name == b.Name { + return a.Hostname < b.Hostname + } + return a.Name < b.Name + } + return a.City < b.City + } + return a.Region < b.Region + } + return a.Country < b.Country + }) +} diff --git a/internal/updater/providers/protonvpn/string.go b/internal/updater/providers/protonvpn/string.go new file mode 100644 index 00000000..a5c6d540 --- /dev/null +++ b/internal/updater/providers/protonvpn/string.go @@ -0,0 +1,14 @@ +package protonvpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.ProtonvpnServer) (s string) { + s = "func ProtonvpnServers() []models.ProtonvpnServer {\n" + s += " return []models.ProtonvpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/purevpn/hosttoserver.go b/internal/updater/providers/purevpn/hosttoserver.go new file mode 100644 index 00000000..0ebd4c12 --- /dev/null +++ b/internal/updater/providers/purevpn/hosttoserver.go @@ -0,0 +1,43 @@ +package purevpn + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.PurevpnServer + +func (hts hostToServer) add(host string) { + // TODO set TCP and UDP compatibility, set hostname + hts[host] = models.PurevpnServer{} +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toServersSlice() (servers []models.PurevpnServer) { + servers = make([]models.PurevpnServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/purevpn/locationtoserver.go b/internal/updater/providers/purevpn/locationtoserver.go new file mode 100644 index 00000000..4e34caf4 --- /dev/null +++ b/internal/updater/providers/purevpn/locationtoserver.go @@ -0,0 +1,33 @@ +package purevpn + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type locationToServer map[string]models.PurevpnServer + +func locationKey(country, region, city string) string { + return country + region + city +} + +func (lts locationToServer) add(country, region, city string, ips []net.IP) { + key := locationKey(country, region, city) + server, ok := lts[key] + if !ok { + server.Country = country + server.Region = region + server.City = city + } + server.IPs = append(server.IPs, ips...) + lts[key] = server +} + +func (lts locationToServer) toServersSlice() (servers []models.PurevpnServer) { + servers = make([]models.PurevpnServer, 0, len(lts)) + for _, server := range lts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/purevpn/resolve.go b/internal/updater/providers/purevpn/resolve.go new file mode 100644 index 00000000..3c7e8469 --- /dev/null +++ b/internal/updater/providers/purevpn/resolve.go @@ -0,0 +1,33 @@ +package purevpn + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxDuration = 20 * time.Second + betweenDuration = time.Second + maxNoNew = 2 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + BetweenDuration: betweenDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/purevpn/servers.go b/internal/updater/providers/purevpn/servers.go new file mode 100644 index 00000000..b5c26211 --- /dev/null +++ b/internal/updater/providers/purevpn/servers.go @@ -0,0 +1,98 @@ +// Package purevpn contains code to obtain the server information +// for the PureVPN provider. +package purevpn + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/publicip" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, + unzipper unzip.Unzipper, presolver resolver.Parallel, minServers int) ( + servers []models.PurevpnServer, warnings []string, err error) { + const url = "https://s3-us-west-1.amazonaws.com/heartbleed/windows/New+OVPN+Files.zip" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + hts.add(host) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + hts.adaptWithIPs(hostToIPs) + + servers = hts.toServersSlice() + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + // Dedup by location + lts := make(locationToServer) + for _, server := range servers { + country, region, city, err := publicip.Info(ctx, client, server.IPs[0]) + if err != nil { + return nil, warnings, err + } + + // TODO split servers by host + lts.add(country, region, city, server.IPs) + } + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + servers = lts.toServersSlice() + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/purevpn/sort.go b/internal/updater/providers/purevpn/sort.go new file mode 100644 index 00000000..98e5e550 --- /dev/null +++ b/internal/updater/providers/purevpn/sort.go @@ -0,0 +1,19 @@ +package purevpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.PurevpnServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + if servers[i].Region == servers[j].Region { + return servers[i].City < servers[j].City + } + return servers[i].Region < servers[j].Region + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/purevpn/string.go b/internal/updater/providers/purevpn/string.go new file mode 100644 index 00000000..48b17054 --- /dev/null +++ b/internal/updater/providers/purevpn/string.go @@ -0,0 +1,14 @@ +package purevpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.PurevpnServer) (s string) { + s = "func PurevpnServers() []models.PurevpnServer {\n" + s += " return []models.PurevpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/surfshark/api.go b/internal/updater/providers/surfshark/api.go new file mode 100644 index 00000000..bdf7ccdb --- /dev/null +++ b/internal/updater/providers/surfshark/api.go @@ -0,0 +1,53 @@ +package surfshark + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") + ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body") +) + +//nolint:unused +type serverData struct { + Host string `json:"connectionName"` + Country string `json:"country"` + Location string `json:"location"` +} + +//nolint:unused,deadcode +func fetchAPI(ctx context.Context, client *http.Client) ( + servers []serverData, err error) { + const url = "https://my.surfshark.com/vpn/api/v4/server/clusters" + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) + } + + decoder := json.NewDecoder(response.Body) + if err := decoder.Decode(&servers); err != nil { + return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) + } + + if err := response.Body.Close(); err != nil { + return nil, err + } + + return servers, nil +} diff --git a/internal/updater/providers/surfshark/hosttoserver.go b/internal/updater/providers/surfshark/hosttoserver.go new file mode 100644 index 00000000..3d4acf04 --- /dev/null +++ b/internal/updater/providers/surfshark/hosttoserver.go @@ -0,0 +1,59 @@ +package surfshark + +import ( + "net" + "strings" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.SurfsharkServer + +func (hts hostToServer) add(host, region string) { + // TODO set TCP and UDP + // TODO set hostname + server, ok := hts[host] + if !ok { + server.Region = region + hts[host] = server + } +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) toSubdomainsSlice() (subdomains []string) { + subdomains = make([]string, 0, len(hts)) + const suffix = ".prod.surfshark.com" + for host := range hts { + subdomain := strings.TrimSuffix(host, suffix) + subdomains = append(subdomains, subdomain) + } + return subdomains +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toServersSlice() (servers []models.SurfsharkServer) { + servers = make([]models.SurfsharkServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/surfshark/regions.go b/internal/updater/providers/surfshark/regions.go new file mode 100644 index 00000000..4f98fb99 --- /dev/null +++ b/internal/updater/providers/surfshark/regions.go @@ -0,0 +1,200 @@ +package surfshark + +import ( + "errors" + "fmt" + "strings" +) + +var ( + errSuffixNotFound = errors.New("suffix not found") + errSubdomainNotFound = errors.New("subdomain not found in subdomain to region mapping") +) + +func parseHost(host string, subdomainToRegion map[string]string) ( + region string, err error) { + const suffix = ".prod.surfshark.com" + if !strings.HasSuffix(host, suffix) { + return "", fmt.Errorf("%w: %s in %s", + errSuffixNotFound, suffix, host) + } + + subdomain := strings.TrimSuffix(host, suffix) + region, ok := subdomainToRegion[subdomain] + if !ok { + return "", fmt.Errorf("%w: %s", errSubdomainNotFound, subdomain) + } + + return region, nil +} + +func subdomainToRegion() (mapping map[string]string) { + return map[string]string{ + "ae-dub": "United Arab Emirates", + "al-tia": "Albania", + "at-vie": "Austria", + "au-adl": "Australia Adelaide", + "au-bne": "Australia Brisbane", + "au-mel": "Australia Melbourne", + "au-per": "Australia Perth", + "au-syd": "Australia Sydney", + "au-us": "Australia US", + "az-bak": "Azerbaijan", + "ba-sjj": "Bosnia and Herzegovina", + "be-bru": "Belgium", + "bg-sof": "Bulgaria", + "br-sao": "Brazil", + "ca-mon": "Canada Montreal", + "ca-tor": "Canada Toronto", + "ca-us": "Canada US", + "ca-van": "Canada Vancouver", + "ch-zur": "Switzerland", + "cl-san": "Chile", + "co-bog": "Colombia", + "cr-sjn": "Costa Rica", + "cy-nic": "Cyprus", + "cz-prg": "Czech Republic", + "de-ber": "Germany Berlin", + "de-fra": "Germany Frankfurt am Main", + "de-fra-st001": "Germany Frankfurt am Main st001", + "de-fra-st002": "Germany Frankfurt am Main st002", + "de-fra-st003": "Germany Frankfurt am Main st003", + "de-fra-st004": "Germany Frankfurt am Main st004", + "de-fra-st005": "Germany Frankfurt am Main st005", + "de-muc": "Germany Munich", + "de-nue": "Germany Nuremberg", + "de-sg": "Germany Singapour", + "de-uk": "Germany UK", + "dk-cph": "Denmark", + "ee-tll": "Estonia", + "es-bcn": "Spain Barcelona", + "es-mad": "Spain Madrid", + "es-vlc": "Spain Valencia", + "fi-hel": "Finland", + "fr-bod": "France Bordeaux", + "fr-mrs": "France Marseilles", + "fr-par": "France Paris", + "fr-se": "France Sweden", + "gr-ath": "Greece", + "hk-hkg": "Hong Kong", + "hr-zag": "Croatia", + "hu-bud": "Hungary", + "id-jak": "Indonesia", + "ie-dub": "Ireland", + "il-tlv": "Israel", + "in-chn": "India Chennai", + "in-idr": "India Indore", + "in-mum": "India Mumbai", + "in-uk": "India UK", + "is-rkv": "Iceland", + "it-mil": "Italy Milan", + "it-rom": "Italy Rome", + "jp-tok": "Japan Tokyo", + "jp-tok-st001": "Japan Tokyo st001", + "jp-tok-st002": "Japan Tokyo st002", + "jp-tok-st003": "Japan Tokyo st003", + "jp-tok-st004": "Japan Tokyo st004", + "jp-tok-st005": "Japan Tokyo st005", + "jp-tok-st006": "Japan Tokyo st006", + "jp-tok-st007": "Japan Tokyo st007", + "jp-tok-st008": "Japan Tokyo st008", + "jp-tok-st009": "Japan Tokyo st009", + "jp-tok-st010": "Japan Tokyo st010", + "jp-tok-st011": "Japan Tokyo st011", + "jp-tok-st012": "Japan Tokyo st012", + "jp-tok-st013": "Japan Tokyo st013", + "kr-seo": "Korea", + "kz-ura": "Kazakhstan", + "lu-ste": "Luxembourg", + "lv-rig": "Latvia", + "ly-tip": "Libya", + "md-chi": "Moldova", + "mk-skp": "North Macedonia", + "my-kul": "Malaysia", + "ng-lag": "Nigeria", + "nl-ams": "Netherlands Amsterdam", + "nl-ams-st001": "Netherlands Amsterdam st001", + "nl-us": "Netherlands US", + "no-osl": "Norway", + "nz-akl": "New Zealand", + "ph-mnl": "Philippines", + "pl-gdn": "Poland Gdansk", + "pl-waw": "Poland Warsaw", + "pt-lis": "Portugal Lisbon", + "pt-lou": "Portugal Loule", + "pt-opo": "Portugal Porto", + "py-asu": "Paraguay", + "ro-buc": "Romania", + "rs-beg": "Serbia", + "ru-mos": "Russia Moscow", + "ru-spt": "Russia St. Petersburg", + "se-sto": "Sweden", + "sg-hk": "Singapore Hong Kong", + "sg-nl": "Singapore Netherlands", + "sg-sng": "Singapore", + "sg-in": "Singapore in", + "sg-sng-st001": "Singapore st001", + "sg-sng-st002": "Singapore st002", + "sg-sng-st003": "Singapore st003", + "sg-sng-st004": "Singapore st004", + "sg-sng-mp001": "Singapore mp001", + "si-lju": "Slovenia", + "sk-bts": "Slovekia", + "th-bkk": "Thailand", + "tr-bur": "Turkey", + "tw-tai": "Taiwan", + "ua-iev": "Ukraine", + "uk-de": "UK Germany", + "uk-fr": "UK France", + "uk-gla": "UK Glasgow", + "uk-lon": "UK London", + "uk-lon-mp001": "UK London mp001", + "uk-lon-st001": "UK London st001", + "uk-lon-st002": "UK London st002", + "uk-lon-st003": "UK London st003", + "uk-lon-st004": "UK London st004", + "uk-lon-st005": "UK London st005", + "uk-man": "UK Manchester", + "us-atl": "US Atlanta", + "us-bdn": "US Bend", + "us-bos": "US Boston", + "us-buf": "US Buffalo", + "us-chi": "US Chicago", + "us-clt": "US Charlotte", + "us-dal": "US Dallas", + "us-den": "US Denver", + "us-dtw": "US Gahanna", + "us-hou": "US Houston", + "us-kan": "US Kansas City", + "us-las": "US Las Vegas", + "us-lax": "US Los Angeles", + "us-ltm": "US Latham", + "us-mia": "US Miami", + "us-mnz": "US Maryland", + "us-nl": "US Netherlands", + "us-nyc": "US New York City", + "us-nyc-mp001": "US New York City mp001", + "us-nyc-st001": "US New York City st001", + "us-nyc-st002": "US New York City st002", + "us-nyc-st003": "US New York City st003", + "us-nyc-st004": "US New York City st004", + "us-nyc-st005": "US New York City st005", + "us-orl": "US Orlando", + "us-phx": "US Phoenix", + "us-pt": "US Portugal", + "us-sea": "US Seatle", + "us-sfo": "US San Francisco", + "us-slc": "US Salt Lake City", + "us-stl": "US Saint Louis", + "us-tpa": "US Tampa", + "vn-hcm": "Vietnam", + "za-jnb": "South Africa", + "ar-bua": "Argentina Buenos Aires", + "tr-ist": "Turkey Istanbul", + "mx-mex": "Mexico City Mexico", + "ca-tor-mp001": "Canada Toronto mp001", + "de-fra-mp001": "Germany Frankfurt mp001", + "nl-ams-mp001": "Netherlands Amsterdam mp001", + "us-sfo-mp001": "US San Francisco mp001", + } +} diff --git a/internal/updater/providers/surfshark/resolve.go b/internal/updater/providers/surfshark/resolve.go new file mode 100644 index 00000000..5d0865b9 --- /dev/null +++ b/internal/updater/providers/surfshark/resolve.go @@ -0,0 +1,32 @@ +package surfshark + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxDuration = 20 * time.Second + betweenDuration = time.Second + maxNoNew = 2 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: maxDuration, + BetweenDuration: betweenDuration, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/surfshark/servers.go b/internal/updater/providers/surfshark/servers.go new file mode 100644 index 00000000..47297713 --- /dev/null +++ b/internal/updater/providers/surfshark/servers.go @@ -0,0 +1,130 @@ +// Package surfshark contains code to obtain the server information +// for the Surshark provider. +package surfshark + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, + presolver resolver.Parallel, minServers int) ( + servers []models.SurfsharkServer, warnings []string, err error) { + const url = "https://my.surfshark.com/vpn/api/v1/server/configurations" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + subdomainToRegion := subdomainToRegion() + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + region, err := parseHost(host, subdomainToRegion) + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + warnings = append(warnings, warning) + continue + } + + hts.add(host, region) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + hts.adaptWithIPs(hostToIPs) + + servers = hts.toServersSlice() + + // process subdomain entries in mapping that were not in the Zip file + subdomainsDone := hts.toSubdomainsSlice() + for _, subdomainDone := range subdomainsDone { + delete(subdomainToRegion, subdomainDone) + } + remainingServers, newWarnings, err := getRemainingServers( + ctx, subdomainToRegion, presolver) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + servers = append(servers, remainingServers...) + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} + +func getRemainingServers(ctx context.Context, + subdomainToRegionLeft map[string]string, presolver resolver.Parallel) ( + servers []models.SurfsharkServer, warnings []string, err error) { + hosts := make([]string, 0, len(subdomainToRegionLeft)) + const suffix = ".prod.surfshark.com" + for subdomain := range subdomainToRegionLeft { + hosts = append(hosts, subdomain+suffix) + } + + const minServers = 0 + hostToIPs, warnings, err := resolveHosts(ctx, presolver, hosts, minServers) + if err != nil { + return nil, warnings, err + } + + servers = make([]models.SurfsharkServer, 0, len(hostToIPs)) + for host, IPs := range hostToIPs { + region, err := parseHost(host, subdomainToRegionLeft) + if err != nil { + return nil, warnings, err + } + server := models.SurfsharkServer{ + Region: region, + IPs: IPs, + } + servers = append(servers, server) + } + + return servers, warnings, nil +} diff --git a/internal/updater/providers/surfshark/sort.go b/internal/updater/providers/surfshark/sort.go new file mode 100644 index 00000000..e8e3c09e --- /dev/null +++ b/internal/updater/providers/surfshark/sort.go @@ -0,0 +1,13 @@ +package surfshark + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.SurfsharkServer) { + sort.Slice(servers, func(i, j int) bool { + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/surfshark/string.go b/internal/updater/providers/surfshark/string.go new file mode 100644 index 00000000..5234f146 --- /dev/null +++ b/internal/updater/providers/surfshark/string.go @@ -0,0 +1,14 @@ +package surfshark + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.SurfsharkServer) (s string) { + s = "func SurfsharkServers() []models.SurfsharkServer {\n" + s += " return []models.SurfsharkServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/torguard/filename.go b/internal/updater/providers/torguard/filename.go new file mode 100644 index 00000000..cc59035a --- /dev/null +++ b/internal/updater/providers/torguard/filename.go @@ -0,0 +1,31 @@ +package torguard + +import "strings" + +func parseFilename(fileName string) (country, city string) { + const prefix = "TorGuard." + const suffix = ".ovpn" + s := strings.TrimPrefix(fileName, prefix) + s = strings.TrimSuffix(s, suffix) + + switch { + case strings.Count(s, ".") == 1 && !strings.HasPrefix(s, "USA"): + parts := strings.Split(s, ".") + country = parts[0] + city = parts[1] + + case strings.HasPrefix(s, "USA"): + country = "USA" + s = strings.TrimPrefix(s, "USA-") + s = strings.ReplaceAll(s, "-", " ") + s = strings.ReplaceAll(s, ".", " ") + s = strings.ToLower(s) + s = strings.Title(s) + city = s + + default: + country = s + } + + return country, city +} diff --git a/internal/updater/providers/torguard/servers.go b/internal/updater/providers/torguard/servers.go new file mode 100644 index 00000000..6e3e0a38 --- /dev/null +++ b/internal/updater/providers/torguard/servers.go @@ -0,0 +1,76 @@ +// Package torguard contains code to obtain the server information +// for the Torguard provider. +package torguard + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, minServers int) ( + servers []models.TorguardServer, warnings []string, err error) { + const url = "https://torguard.net/downloads/OpenVPN-TCP-Linux.zip" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + servers = make([]models.TorguardServer, 0, len(contents)) + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + country, city := parseFilename(fileName) + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + ip, warning, err := openvpn.ExtractIP(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + server := models.TorguardServer{ + Country: country, + City: city, + Hostname: host, + IP: ip, + } + servers = append(servers, server) + } + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/torguard/sort.go b/internal/updater/providers/torguard/sort.go new file mode 100644 index 00000000..c66542c3 --- /dev/null +++ b/internal/updater/providers/torguard/sort.go @@ -0,0 +1,19 @@ +package torguard + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.TorguardServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Country == servers[j].Country { + if servers[i].City == servers[j].City { + return servers[i].Hostname < servers[j].Hostname + } + return servers[i].City < servers[j].City + } + return servers[i].Country < servers[j].Country + }) +} diff --git a/internal/updater/providers/torguard/string.go b/internal/updater/providers/torguard/string.go new file mode 100644 index 00000000..9bcebbf7 --- /dev/null +++ b/internal/updater/providers/torguard/string.go @@ -0,0 +1,14 @@ +package torguard + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.TorguardServer) (s string) { + s = "func TorguardServers() []models.TorguardServer {\n" + s += " return []models.TorguardServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/vyprvpn/filename.go b/internal/updater/providers/vyprvpn/filename.go new file mode 100644 index 00000000..b2caa712 --- /dev/null +++ b/internal/updater/providers/vyprvpn/filename.go @@ -0,0 +1,22 @@ +package vyprvpn + +import ( + "errors" + "fmt" + "strings" +) + +var errNotOvpnExt = errors.New("filename does not have the openvpn file extension") + +func parseFilename(fileName string) ( + region string, err error, +) { + const suffix = ".ovpn" + if !strings.HasSuffix(fileName, suffix) { + return "", fmt.Errorf("%w: %s", errNotOvpnExt, fileName) + } + + region = strings.TrimSuffix(fileName, suffix) + region = strings.ReplaceAll(region, " - ", " ") + return region, nil +} diff --git a/internal/updater/providers/vyprvpn/hosttoserver.go b/internal/updater/providers/vyprvpn/hosttoserver.go new file mode 100644 index 00000000..f880f3e5 --- /dev/null +++ b/internal/updater/providers/vyprvpn/hosttoserver.go @@ -0,0 +1,47 @@ +package vyprvpn + +import ( + "net" + + "github.com/qdm12/gluetun/internal/models" +) + +type hostToServer map[string]models.VyprvpnServer + +func (hts hostToServer) add(host, region string) { + server, ok := hts[host] + // TODO set host + if !ok { + server.Region = region + hts[host] = server + } +} + +func (hts hostToServer) toHostsSlice() (hosts []string) { + hosts = make([]string, 0, len(hts)) + for host := range hts { + hosts = append(hosts, host) + } + return hosts +} + +func (hts hostToServer) adaptWithIPs(hostToIPs map[string][]net.IP) { + for host, IPs := range hostToIPs { + server := hts[host] + server.IPs = IPs + hts[host] = server + } + for host, server := range hts { + if len(server.IPs) == 0 { + delete(hts, host) + } + } +} + +func (hts hostToServer) toServersSlice() (servers []models.VyprvpnServer) { + servers = make([]models.VyprvpnServer, 0, len(hts)) + for _, server := range hts { + servers = append(servers, server) + } + return servers +} diff --git a/internal/updater/providers/vyprvpn/resolve.go b/internal/updater/providers/vyprvpn/resolve.go new file mode 100644 index 00000000..b5e2175a --- /dev/null +++ b/internal/updater/providers/vyprvpn/resolve.go @@ -0,0 +1,30 @@ +package vyprvpn + +import ( + "context" + "net" + "time" + + "github.com/qdm12/gluetun/internal/updater/resolver" +) + +func resolveHosts(ctx context.Context, presolver resolver.Parallel, + hosts []string, minServers int) (hostToIPs map[string][]net.IP, + warnings []string, err error) { + const ( + maxFailRatio = 0.1 + maxNoNew = 2 + maxFails = 2 + ) + settings := resolver.ParallelSettings{ + MaxFailRatio: maxFailRatio, + MinFound: minServers, + Repeat: resolver.RepeatSettings{ + MaxDuration: time.Second, + MaxNoNew: maxNoNew, + MaxFails: maxFails, + SortIPs: true, + }, + } + return presolver.Resolve(ctx, hosts, settings) +} diff --git a/internal/updater/providers/vyprvpn/servers.go b/internal/updater/providers/vyprvpn/servers.go new file mode 100644 index 00000000..120dd700 --- /dev/null +++ b/internal/updater/providers/vyprvpn/servers.go @@ -0,0 +1,82 @@ +// Package vyprvpn contains code to obtain the server information +// for the VyprVPN provider. +package vyprvpn + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/updater/openvpn" + "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, unzipper unzip.Unzipper, + presolver resolver.Parallel, minServers int) ( + servers []models.VyprvpnServer, warnings []string, err error) { + const url = "https://support.vyprvpn.com/hc/article_attachments/360052617332/Vypr_OpenVPN_20200320.zip" + contents, err := unzipper.FetchAndExtract(ctx, url) + if err != nil { + return nil, nil, err + } else if len(contents) < minServers { + return nil, nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(contents), minServers) + } + + hts := make(hostToServer) + + for fileName, content := range contents { + if !strings.HasSuffix(fileName, ".ovpn") { + continue // not an OpenVPN file + } + + host, warning, err := openvpn.ExtractHost(content) + if warning != "" { + warnings = append(warnings, warning) + } + if err != nil { + // treat error as warning and go to next file + warning := err.Error() + " in " + fileName + warnings = append(warnings, warning) + continue + } + + region, err := parseFilename(fileName) + if err != nil { + warnings = append(warnings, err.Error()) + continue + } + + hts.add(host, region) + } + + if len(hts) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(hts), minServers) + } + + hosts := hts.toHostsSlice() + hostToIPs, newWarnings, err := resolveHosts(ctx, presolver, hosts, minServers) + warnings = append(warnings, newWarnings...) + if err != nil { + return nil, warnings, err + } + + hts.adaptWithIPs(hostToIPs) + + servers = hts.toServersSlice() + + if len(servers) < minServers { + return nil, warnings, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, warnings, nil +} diff --git a/internal/updater/providers/vyprvpn/sort.go b/internal/updater/providers/vyprvpn/sort.go new file mode 100644 index 00000000..fea4641e --- /dev/null +++ b/internal/updater/providers/vyprvpn/sort.go @@ -0,0 +1,13 @@ +package vyprvpn + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.VyprvpnServer) { + sort.Slice(servers, func(i, j int) bool { + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/vyprvpn/string.go b/internal/updater/providers/vyprvpn/string.go new file mode 100644 index 00000000..da1b477d --- /dev/null +++ b/internal/updater/providers/vyprvpn/string.go @@ -0,0 +1,14 @@ +package vyprvpn + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.VyprvpnServer) (s string) { + s = "func VyprvpnServers() []models.VyprvpnServer {\n" + s += " return []models.VyprvpnServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/providers/windscribe/api.go b/internal/updater/providers/windscribe/api.go new file mode 100644 index 00000000..84a6acc9 --- /dev/null +++ b/internal/updater/providers/windscribe/api.go @@ -0,0 +1,61 @@ +package windscribe + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "strconv" + "time" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") + ErrUnmarshalResponseBody = errors.New("failed unmarshaling response body") +) + +type apiData struct { + Data []regionData `json:"data"` +} + +type regionData struct { + Region string `json:"name"` + Groups []groupData `json:"groups"` +} + +type groupData struct { + City string `json:"city"` + Nodes []serverData `json:"nodes"` +} + +type serverData struct { + Hostname string `json:"hostname"` + OpenvpnIP net.IP `json:"ip2"` +} + +func fetchAPI(ctx context.Context, client *http.Client) ( + data apiData, err error) { + const baseURL = "https://assets.windscribe.com/serverlist/mob-v2/1/" + cacheBreaker := time.Now().Unix() + url := baseURL + strconv.Itoa(int(cacheBreaker)) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return data, err + } + + response, err := client.Do(request) + if err != nil { + return data, err + } + defer response.Body.Close() + + decoder := json.NewDecoder(response.Body) + if err := decoder.Decode(&data); err != nil { + return data, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) + } + + return data, nil +} diff --git a/internal/updater/providers/windscribe/servers.go b/internal/updater/providers/windscribe/servers.go new file mode 100644 index 00000000..3a0de740 --- /dev/null +++ b/internal/updater/providers/windscribe/servers.go @@ -0,0 +1,47 @@ +// Package windscribe contains code to obtain the server information +// for the Windscribe provider. +package windscribe + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/qdm12/gluetun/internal/models" +) + +var ErrNotEnoughServers = errors.New("not enough servers found") + +func GetServers(ctx context.Context, client *http.Client, minServers int) ( + servers []models.WindscribeServer, err error) { + data, err := fetchAPI(ctx, client) + if err != nil { + return nil, err + } + + for _, regionData := range data.Data { + region := regionData.Region + for _, group := range regionData.Groups { + city := group.City + for _, node := range group.Nodes { + server := models.WindscribeServer{ + Region: region, + City: city, + Hostname: node.Hostname, + IP: node.OpenvpnIP, + } + servers = append(servers, server) + } + } + } + + if len(servers) < minServers { + return nil, fmt.Errorf("%w: %d and expected at least %d", + ErrNotEnoughServers, len(servers), minServers) + } + + sortServers(servers) + + return servers, nil +} diff --git a/internal/updater/providers/windscribe/sort.go b/internal/updater/providers/windscribe/sort.go new file mode 100644 index 00000000..a0c8861b --- /dev/null +++ b/internal/updater/providers/windscribe/sort.go @@ -0,0 +1,19 @@ +package windscribe + +import ( + "sort" + + "github.com/qdm12/gluetun/internal/models" +) + +func sortServers(servers []models.WindscribeServer) { + sort.Slice(servers, func(i, j int) bool { + if servers[i].Region == servers[j].Region { + if servers[i].City == servers[j].City { + return servers[i].Hostname < servers[j].Hostname + } + return servers[i].City < servers[j].City + } + return servers[i].Region < servers[j].Region + }) +} diff --git a/internal/updater/providers/windscribe/string.go b/internal/updater/providers/windscribe/string.go new file mode 100644 index 00000000..7e172c87 --- /dev/null +++ b/internal/updater/providers/windscribe/string.go @@ -0,0 +1,14 @@ +package windscribe + +import "github.com/qdm12/gluetun/internal/models" + +func Stringify(servers []models.WindscribeServer) (s string) { + s = "func WindscribeServers() []models.WindscribeServer {\n" + s += " return []models.WindscribeServer{\n" + for _, server := range servers { + s += " " + server.String() + ",\n" + } + s += " }\n" + s += "}" + return s +} diff --git a/internal/updater/purevpn.go b/internal/updater/purevpn.go deleted file mode 100644 index b3a5e9ba..00000000 --- a/internal/updater/purevpn.go +++ /dev/null @@ -1,136 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net/http" - "sort" - "strings" - "time" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/publicip" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updatePurevpn(ctx context.Context) (err error) { - servers, warnings, err := findPurevpnServers(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("PureVPN: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Purevpn servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyPurevpnServers(servers)) - } - u.servers.Purevpn.Timestamp = u.timeNow().Unix() - u.servers.Purevpn.Servers = servers - return nil -} - -func findPurevpnServers(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.PurevpnServer, warnings []string, err error) { - const zipURL = "https://s3-us-west-1.amazonaws.com/heartbleed/windows/New+OVPN+Files.zip" - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - hosts := make([]string, 0, len(contents)) - for fileName, content := range contents { - if strings.HasSuffix(fileName, "-tcp.ovpn") { - continue // only parse UDP files - } - host, warning, err := extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - return nil, warnings, fmt.Errorf("%w in %q", err, fileName) - } - hosts = append(hosts, host) - } - - const ( - maxFailRatio = 0.1 - maxDuration = 20 * time.Second - betweenDuration = time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - uniqueServers := make(map[string]models.PurevpnServer, len(hostToIPs)) - for host, IPs := range hostToIPs { - if len(IPs) == 0 { - warning := fmt.Sprintf("no IP address found for host %q", host) - warnings = append(warnings, warning) - continue - } - - country, region, city, err := publicip.Info(ctx, client, IPs[0]) - if err != nil { - return nil, warnings, err - } - key := country + region + city - server, ok := uniqueServers[key] - if ok { - server.IPs = append(server.IPs, IPs...) - } else { - server = models.PurevpnServer{ - Country: country, - Region: region, - City: city, - IPs: IPs, - } - } - uniqueServers[key] = server - } - - servers = make([]models.PurevpnServer, len(uniqueServers)) - i := 0 - for _, server := range uniqueServers { - servers[i] = server - i++ - } - - sort.Slice(servers, func(i, j int) bool { - if servers[i].Country == servers[j].Country { - if servers[i].Region == servers[j].Region { - return servers[i].City < servers[j].City - } - return servers[i].Region < servers[j].Region - } - return servers[i].Country < servers[j].Country - }) - - return servers, warnings, nil -} - -func stringifyPurevpnServers(servers []models.PurevpnServer) (s string) { - s = "func PurevpnServers() []models.PurevpnServer {\n" - s += " return []models.PurevpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/surfshark.go b/internal/updater/surfshark.go deleted file mode 100644 index 93cfd74e..00000000 --- a/internal/updater/surfshark.go +++ /dev/null @@ -1,417 +0,0 @@ -package updater - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "sort" - "strings" - "time" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updateSurfshark(ctx context.Context) (err error) { - servers, warnings, err := findSurfsharkServersFromZip(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Surfshark: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Surfshark servers: %w", err) - } - if u.options.Stdout { - u.println(stringifySurfsharkServers(servers)) - } - u.servers.Surfshark.Timestamp = u.timeNow().Unix() - u.servers.Surfshark.Servers = servers - return nil -} - -//nolint:deadcode,unused -func findSurfsharkServersFromAPI(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.SurfsharkServer, warnings []string, err error) { - const url = "https://my.surfshark.com/vpn/api/v4/server/clusters" - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - decoder := json.NewDecoder(response.Body) - var jsonServers []struct { - Host string `json:"connectionName"` - Country string `json:"country"` - Location string `json:"location"` - } - if err := decoder.Decode(&jsonServers); err != nil { - return nil, nil, err - } - - if err := response.Body.Close(); err != nil { - return nil, nil, err - } - - hosts := make([]string, len(jsonServers)) - for i := range jsonServers { - hosts[i] = jsonServers[i].Host - } - - const ( - maxFailRatio = 0.1 - maxDuration = 20 * time.Second - betweenDuration = time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - }, - } - hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings) - if err != nil { - return nil, warnings, err - } - - for _, jsonServer := range jsonServers { - host := jsonServer.Host - IPs := hostToIPs[host] - if len(IPs) == 0 { - warning := fmt.Sprintf("no IP address found for host %q", host) - warnings = append(warnings, warning) - continue - } - server := models.SurfsharkServer{ - Region: jsonServer.Country + " " + jsonServer.Location, - IPs: IPs, - } - servers = append(servers, server) - } - return servers, warnings, nil -} - -func findSurfsharkServersFromZip(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.SurfsharkServer, warnings []string, err error) { - const zipURL = "https://my.surfshark.com/vpn/api/v1/server/configurations" - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - mapping := surfsharkSubdomainToRegion() - hosts := make([]string, 0, len(contents)) - for fileName, content := range contents { - if strings.HasSuffix(fileName, "_tcp.ovpn") { - continue // only parse UDP files - } - host, warning, err := extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - // treat error as warning and go to next file - warnings = append(warnings, err.Error()+" in "+fileName) - continue - } - hosts = append(hosts, host) - } - - const ( - maxFailRatio = 0.1 - maxDuration = 20 * time.Second - betweenDuration = time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - for host, IPs := range hostToIPs { - if len(IPs) == 0 { - warning := fmt.Sprintf("no IP address found for host %q", host) - warnings = append(warnings, warning) - continue - } - subdomain := strings.TrimSuffix(host, ".prod.surfshark.com") - region, ok := mapping[subdomain] - if ok { - delete(mapping, subdomain) - } else { - region = strings.TrimSuffix(host, ".prod.surfshark.com") - warning := fmt.Sprintf("subdomain %q not found in Surfshark mapping", subdomain) - warnings = append(warnings, warning) - } - server := models.SurfsharkServer{ - Region: region, - IPs: IPs, - } - servers = append(servers, server) - } - - // process entries in mapping that were not in zip file - remainingServers, newWarnings, err := getRemainingServers(ctx, mapping, presolver) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - servers = append(servers, remainingServers...) - - sort.Slice(servers, func(i, j int) bool { - return servers[i].Region < servers[j].Region - }) - return servers, warnings, nil -} - -func getRemainingServers(ctx context.Context, mapping map[string]string, presolver resolver.Parallel) ( - servers []models.SurfsharkServer, warnings []string, err error) { - hosts := make([]string, 0, len(mapping)) - for subdomain := range mapping { - hosts = append(hosts, subdomain+".prod.surfshark.com") - } - - const ( - maxFailRatio = 0.3 - maxDuration = 20 * time.Second - betweenDuration = time.Second - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: maxDuration, - BetweenDuration: betweenDuration, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, warnings, err := presolver.Resolve(ctx, hosts, settings) - if err != nil { - return nil, warnings, err - } - - servers = make([]models.SurfsharkServer, 0, len(hostToIPs)) - for host, IPs := range hostToIPs { - subdomain := strings.TrimSuffix(host, ".prod.surfshark.com") - server := models.SurfsharkServer{ - Region: mapping[subdomain], - IPs: IPs, - } - servers = append(servers, server) - } - - return servers, warnings, nil -} - -func stringifySurfsharkServers(servers []models.SurfsharkServer) (s string) { - s = "func SurfsharkServers() []models.SurfsharkServer {\n" - s += " return []models.SurfsharkServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} - -func surfsharkSubdomainToRegion() (mapping map[string]string) { - return map[string]string{ - "ae-dub": "United Arab Emirates", - "al-tia": "Albania", - "at-vie": "Austria", - "au-adl": "Australia Adelaide", - "au-bne": "Australia Brisbane", - "au-mel": "Australia Melbourne", - "au-per": "Australia Perth", - "au-syd": "Australia Sydney", - "au-us": "Australia US", - "az-bak": "Azerbaijan", - "ba-sjj": "Bosnia and Herzegovina", - "be-bru": "Belgium", - "bg-sof": "Bulgaria", - "br-sao": "Brazil", - "ca-mon": "Canada Montreal", - "ca-tor": "Canada Toronto", - "ca-us": "Canada US", - "ca-van": "Canada Vancouver", - "ch-zur": "Switzerland", - "cl-san": "Chile", - "co-bog": "Colombia", - "cr-sjn": "Costa Rica", - "cy-nic": "Cyprus", - "cz-prg": "Czech Republic", - "de-ber": "Germany Berlin", - "de-fra": "Germany Frankfurt am Main", - "de-fra-st001": "Germany Frankfurt am Main st001", - "de-fra-st002": "Germany Frankfurt am Main st002", - "de-fra-st003": "Germany Frankfurt am Main st003", - "de-fra-st004": "Germany Frankfurt am Main st004", - "de-fra-st005": "Germany Frankfurt am Main st005", - "de-muc": "Germany Munich", - "de-nue": "Germany Nuremberg", - "de-sg": "Germany Singapour", - "de-uk": "Germany UK", - "dk-cph": "Denmark", - "ee-tll": "Estonia", - "es-bcn": "Spain Barcelona", - "es-mad": "Spain Madrid", - "es-vlc": "Spain Valencia", - "fi-hel": "Finland", - "fr-bod": "France Bordeaux", - "fr-mrs": "France Marseilles", - "fr-par": "France Paris", - "fr-se": "France Sweden", - "gr-ath": "Greece", - "hk-hkg": "Hong Kong", - "hr-zag": "Croatia", - "hu-bud": "Hungary", - "id-jak": "Indonesia", - "ie-dub": "Ireland", - "il-tlv": "Israel", - "in-chn": "India Chennai", - "in-idr": "India Indore", - "in-mum": "India Mumbai", - "in-uk": "India UK", - "is-rkv": "Iceland", - "it-mil": "Italy Milan", - "it-rom": "Italy Rome", - "jp-tok": "Japan Tokyo", - "jp-tok-st001": "Japan Tokyo st001", - "jp-tok-st002": "Japan Tokyo st002", - "jp-tok-st003": "Japan Tokyo st003", - "jp-tok-st004": "Japan Tokyo st004", - "jp-tok-st005": "Japan Tokyo st005", - "jp-tok-st006": "Japan Tokyo st006", - "jp-tok-st007": "Japan Tokyo st007", - "jp-tok-st008": "Japan Tokyo st008", - "jp-tok-st009": "Japan Tokyo st009", - "jp-tok-st010": "Japan Tokyo st010", - "jp-tok-st011": "Japan Tokyo st011", - "jp-tok-st012": "Japan Tokyo st012", - "jp-tok-st013": "Japan Tokyo st013", - "kr-seo": "Korea", - "kz-ura": "Kazakhstan", - "lu-ste": "Luxembourg", - "lv-rig": "Latvia", - "ly-tip": "Libya", - "md-chi": "Moldova", - "mk-skp": "North Macedonia", - "my-kul": "Malaysia", - "ng-lag": "Nigeria", - "nl-ams": "Netherlands Amsterdam", - "nl-ams-st001": "Netherlands Amsterdam st001", - "nl-us": "Netherlands US", - "no-osl": "Norway", - "nz-akl": "New Zealand", - "ph-mnl": "Philippines", - "pl-gdn": "Poland Gdansk", - "pl-waw": "Poland Warsaw", - "pt-lis": "Portugal Lisbon", - "pt-lou": "Portugal Loule", - "pt-opo": "Portugal Porto", - "py-asu": "Paraguay", - "ro-buc": "Romania", - "rs-beg": "Serbia", - "ru-mos": "Russia Moscow", - "ru-spt": "Russia St. Petersburg", - "se-sto": "Sweden", - "sg-hk": "Singapore Hong Kong", - "sg-nl": "Singapore Netherlands", - "sg-sng": "Singapore", - "sg-in": "Singapore in", - "sg-sng-st001": "Singapore st001", - "sg-sng-st002": "Singapore st002", - "sg-sng-st003": "Singapore st003", - "sg-sng-st004": "Singapore st004", - "sg-sng-mp001": "Singapore mp001", - "si-lju": "Slovenia", - "sk-bts": "Slovekia", - "th-bkk": "Thailand", - "tr-bur": "Turkey", - "tw-tai": "Taiwan", - "ua-iev": "Ukraine", - "uk-de": "UK Germany", - "uk-fr": "UK France", - "uk-gla": "UK Glasgow", - "uk-lon": "UK London", - "uk-lon-mp001": "UK London mp001", - "uk-lon-st001": "UK London st001", - "uk-lon-st002": "UK London st002", - "uk-lon-st003": "UK London st003", - "uk-lon-st004": "UK London st004", - "uk-lon-st005": "UK London st005", - "uk-man": "UK Manchester", - "us-atl": "US Atlanta", - "us-bdn": "US Bend", - "us-bos": "US Boston", - "us-buf": "US Buffalo", - "us-chi": "US Chicago", - "us-clt": "US Charlotte", - "us-dal": "US Dallas", - "us-den": "US Denver", - "us-dtw": "US Gahanna", - "us-hou": "US Houston", - "us-kan": "US Kansas City", - "us-las": "US Las Vegas", - "us-lax": "US Los Angeles", - "us-ltm": "US Latham", - "us-mia": "US Miami", - "us-mnz": "US Maryland", - "us-nl": "US Netherlands", - "us-nyc": "US New York City", - "us-nyc-mp001": "US New York City mp001", - "us-nyc-st001": "US New York City st001", - "us-nyc-st002": "US New York City st002", - "us-nyc-st003": "US New York City st003", - "us-nyc-st004": "US New York City st004", - "us-nyc-st005": "US New York City st005", - "us-orl": "US Orlando", - "us-phx": "US Phoenix", - "us-pt": "US Portugal", - "us-sea": "US Seatle", - "us-sfo": "US San Francisco", - "us-slc": "US Salt Lake City", - "us-stl": "US Saint Louis", - "us-tpa": "US Tampa", - "vn-hcm": "Vietnam", - "za-jnb": "South Africa", - "ar-bua": "Argentina Buenos Aires", - "tr-ist": "Turkey Istanbul", - "mx-mex": "Mexico City Mexico", - "ca-tor-mp001": "Canada Toronto mp001", - "de-fra-mp001": "Germany Frankfurt mp001", - "nl-ams-mp001": "Netherlands Amsterdam mp001", - "us-sfo-mp001": "US San Francisco mp001", - } -} diff --git a/internal/updater/torguard.go b/internal/updater/torguard.go deleted file mode 100644 index 889b1536..00000000 --- a/internal/updater/torguard.go +++ /dev/null @@ -1,114 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net" - "net/http" - "sort" - "strconv" - "strings" - - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updateTorguard(ctx context.Context) (err error) { - servers, warnings, err := findTorguardServersFromZip(ctx, u.client) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Torguard: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Torguard servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyTorguardServers(servers)) - } - u.servers.Torguard.Timestamp = u.timeNow().Unix() - u.servers.Torguard.Servers = servers - return nil -} - -func findTorguardServersFromZip(ctx context.Context, client *http.Client) ( - servers []models.TorguardServer, warnings []string, err error) { - // Note: all servers do both TCP and UDP - const zipURL = "https://torguard.net/downloads/OpenVPN-TCP-Linux.zip" - - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - for fileName, content := range contents { - var server models.TorguardServer - - const prefix = "TorGuard." - const suffix = ".ovpn" - s := strings.TrimPrefix(fileName, prefix) - s = strings.TrimSuffix(s, suffix) - - switch { - case strings.Count(s, ".") == 1 && !strings.HasPrefix(s, "USA"): - parts := strings.Split(s, ".") - server.Country = parts[0] - server.City = parts[1] - case strings.HasPrefix(s, "USA"): - server.Country = "USA" - s = strings.TrimPrefix(s, "USA-") - s = strings.ReplaceAll(s, "-", " ") - s = strings.ReplaceAll(s, ".", " ") - s = strings.ToLower(s) - s = strings.Title(s) - server.City = s - default: - server.Country = s - } - - hostnames := extractRemoteHostsFromOpenvpn(content, true, false) - if len(hostnames) != 1 { - warning := "found " + strconv.Itoa(len(hostnames)) + - " hostname(s) instead of 1 in " + fileName - warnings = append(warnings, warning) - continue - } - server.Hostname = hostnames[0] - - IPs := extractRemoteHostsFromOpenvpn(content, false, true) - if len(IPs) != 1 { - warning := "found " + strconv.Itoa(len(IPs)) + - " IP(s) instead of 1 in " + fileName - warnings = append(warnings, warning) - continue - } - server.IP = net.ParseIP(IPs[0]) - if server.IP == nil { - warnings = append(warnings, "IP address "+IPs[0]+" is not valid in file "+fileName) - } - - servers = append(servers, server) - } - - sort.Slice(servers, func(i, j int) bool { - if servers[i].Country == servers[j].Country { - if servers[i].City == servers[j].City { - return servers[i].Hostname < servers[j].Hostname - } - return servers[i].City < servers[j].City - } - return servers[i].Country < servers[j].Country - }) - - return servers, warnings, nil -} - -func stringifyTorguardServers(servers []models.TorguardServer) (s string) { - s = "func TorguardServers() []models.TorguardServer {\n" - s += " return []models.TorguardServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/unzip/extract.go b/internal/updater/unzip/extract.go new file mode 100644 index 00000000..87bbbf9b --- /dev/null +++ b/internal/updater/unzip/extract.go @@ -0,0 +1,36 @@ +package unzip + +import ( + "archive/zip" + "bytes" + "io/ioutil" + "path/filepath" + "strings" +) + +func zipExtractAll(zipBytes []byte) (contents map[string][]byte, err error) { + r, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes))) + if err != nil { + return nil, err + } + contents = map[string][]byte{} + for _, zf := range r.File { + fileName := filepath.Base(zf.Name) + if !strings.HasSuffix(fileName, ".ovpn") { + continue + } + f, err := zf.Open() + if err != nil { + return nil, err + } + defer f.Close() + contents[fileName], err = ioutil.ReadAll(f) + if err != nil { + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + } + return contents, nil +} diff --git a/internal/updater/unzip/fetch.go b/internal/updater/unzip/fetch.go new file mode 100644 index 00000000..8fdb1920 --- /dev/null +++ b/internal/updater/unzip/fetch.go @@ -0,0 +1,52 @@ +package unzip + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "net/http" +) + +var ( + ErrHTTPStatusCodeNotOK = errors.New("HTTP status code not OK") +) + +func (u *unzipper) FetchAndExtract(ctx context.Context, url string) ( + contents map[string][]byte, err error) { + contents = make(map[string][]byte) + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := u.client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) + } + + b, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if err := response.Body.Close(); err != nil { + return nil, err + } + + newContents, err := zipExtractAll(b) + if err != nil { + return nil, err + } + for fileName, content := range newContents { + contents[fileName] = content + } + + return contents, nil +} diff --git a/internal/updater/unzip/unzip.go b/internal/updater/unzip/unzip.go new file mode 100644 index 00000000..2b638149 --- /dev/null +++ b/internal/updater/unzip/unzip.go @@ -0,0 +1,22 @@ +// Package unzip defines the Unzipper which fetches and extract a zip file +// containing multiple files. +package unzip + +import ( + "context" + "net/http" +) + +type Unzipper interface { + FetchAndExtract(ctx context.Context, url string) (contents map[string][]byte, err error) +} + +type unzipper struct { + client *http.Client +} + +func New(client *http.Client) Unzipper { + return &unzipper{ + client: client, + } +} diff --git a/internal/updater/updater.go b/internal/updater/updater.go index 2562583b..da40a536 100644 --- a/internal/updater/updater.go +++ b/internal/updater/updater.go @@ -10,6 +10,7 @@ import ( "github.com/qdm12/gluetun/internal/configuration" "github.com/qdm12/gluetun/internal/models" "github.com/qdm12/gluetun/internal/updater/resolver" + "github.com/qdm12/gluetun/internal/updater/unzip" "github.com/qdm12/golibs/logging" ) @@ -30,6 +31,7 @@ type updater struct { println func(s string) presolver resolver.Parallel client *http.Client + unzipper unzip.Unzipper } func New(settings configuration.Updater, httpClient *http.Client, @@ -37,12 +39,14 @@ func New(settings configuration.Updater, httpClient *http.Client, if len(settings.DNSAddress) == 0 { settings.DNSAddress = "1.1.1.1" } + unzipper := unzip.New(httpClient) return &updater{ logger: logger, timeNow: time.Now, println: func(s string) { fmt.Println(s) }, presolver: resolver.NewParallelResolver(settings.DNSAddress), client: httpClient, + unzipper: unzipper, options: settings, servers: currentServers, } diff --git a/internal/updater/vyprvpn.go b/internal/updater/vyprvpn.go deleted file mode 100644 index b3ff8156..00000000 --- a/internal/updater/vyprvpn.go +++ /dev/null @@ -1,108 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "net/http" - "sort" - "strings" - "time" - - "github.com/qdm12/gluetun/internal/models" - "github.com/qdm12/gluetun/internal/updater/resolver" -) - -func (u *updater) updateVyprvpn(ctx context.Context) (err error) { - servers, warnings, err := findVyprvpnServers(ctx, u.client, u.presolver) - if u.options.CLI { - for _, warning := range warnings { - u.logger.Warn("Vyprvpn: %s", warning) - } - } - if err != nil { - return fmt.Errorf("cannot update Vyprvpn servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyVyprvpnServers(servers)) - } - u.servers.Vyprvpn.Timestamp = u.timeNow().Unix() - u.servers.Vyprvpn.Servers = servers - return nil -} - -func findVyprvpnServers(ctx context.Context, client *http.Client, presolver resolver.Parallel) ( - servers []models.VyprvpnServer, warnings []string, err error) { - const zipURL = "https://support.vyprvpn.com/hc/article_attachments/360052617332/Vypr_OpenVPN_20200320.zip" - contents, err := fetchAndExtractFiles(ctx, client, zipURL) - if err != nil { - return nil, nil, err - } - - hostToRegion := make(map[string]string, len(contents)) - for fileName, content := range contents { - if err := ctx.Err(); err != nil { - return nil, warnings, err - } - host, warning, err := extractHostFromOVPN(content) - if len(warning) > 0 { - warnings = append(warnings, warning) - } - if err != nil { - return nil, warnings, fmt.Errorf("%w in %s", err, fileName) - } - region := strings.TrimSuffix(fileName, ".ovpn") - region = strings.ReplaceAll(region, " - ", " ") - hostToRegion[host] = region - } - - hosts := make([]string, len(hostToRegion)) - i := 0 - for host := range hostToRegion { - hosts[i] = host - i++ - } - - const ( - maxFailRatio = 0.1 - maxNoNew = 2 - maxFails = 2 - ) - settings := resolver.ParallelSettings{ - MaxFailRatio: maxFailRatio, - Repeat: resolver.RepeatSettings{ - MaxDuration: time.Second, - MaxNoNew: maxNoNew, - MaxFails: maxFails, - SortIPs: true, - }, - } - hostToIPs, newWarnings, err := presolver.Resolve(ctx, hosts, settings) - warnings = append(warnings, newWarnings...) - if err != nil { - return nil, warnings, err - } - - servers = make([]models.VyprvpnServer, 0, len(hostToIPs)) - for host, IPs := range hostToIPs { - server := models.VyprvpnServer{ - Region: hostToRegion[host], - IPs: IPs, - } - servers = append(servers, server) - } - sort.Slice(servers, func(i, j int) bool { - return servers[i].Region < servers[j].Region - }) - return servers, warnings, nil -} - -func stringifyVyprvpnServers(servers []models.VyprvpnServer) (s string) { - s = "func VyprvpnServers() []models.VyprvpnServer {\n" - s += " return []models.VyprvpnServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/windscribe.go b/internal/updater/windscribe.go deleted file mode 100644 index f0a052ab..00000000 --- a/internal/updater/windscribe.go +++ /dev/null @@ -1,100 +0,0 @@ -package updater - -import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" - "sort" - "time" - - "github.com/qdm12/gluetun/internal/models" -) - -func (u *updater) updateWindscribe(ctx context.Context) (err error) { - servers, err := findWindscribeServers(ctx, u.client) - if err != nil { - return fmt.Errorf("cannot update Windscribe servers: %w", err) - } - if u.options.Stdout { - u.println(stringifyWindscribeServers(servers)) - } - u.servers.Windscribe.Timestamp = u.timeNow().Unix() - u.servers.Windscribe.Servers = servers - return nil -} - -func findWindscribeServers(ctx context.Context, client *http.Client) (servers []models.WindscribeServer, err error) { - const baseURL = "https://assets.windscribe.com/serverlist/mob-v2/1/" - cacheBreaker := time.Now().Unix() - url := fmt.Sprintf("%s%d", baseURL, cacheBreaker) - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: %s", ErrHTTPStatusCodeNotOK, response.Status) - } - - decoder := json.NewDecoder(response.Body) - var jsonData struct { - Data []struct { - Region string `json:"name"` - Groups []struct { - City string `json:"city"` - Nodes []struct { - Hostname string `json:"hostname"` - OpenvpnIP net.IP `json:"ip2"` - } `json:"nodes"` - } `json:"groups"` - } `json:"data"` - } - if err := decoder.Decode(&jsonData); err != nil { - return nil, fmt.Errorf("%w: %s", ErrUnmarshalResponseBody, err) - } - - if err := response.Body.Close(); err != nil { - return nil, err - } - - for _, regionBlock := range jsonData.Data { - region := regionBlock.Region - for _, group := range regionBlock.Groups { - city := group.City - for _, node := range group.Nodes { - server := models.WindscribeServer{ - Region: region, - City: city, - Hostname: node.Hostname, - IP: node.OpenvpnIP, - } - servers = append(servers, server) - } - } - } - sort.Slice(servers, func(i, j int) bool { - return servers[i].Region+servers[i].City+servers[i].Hostname < - servers[j].Region+servers[j].City+servers[j].Hostname - }) - return servers, nil -} - -func stringifyWindscribeServers(servers []models.WindscribeServer) (s string) { - s = "func WindscribeServers() []models.WindscribeServer {\n" - s += " return []models.WindscribeServer{\n" - for _, server := range servers { - s += " " + server.String() + ",\n" - } - s += " }\n" - s += "}" - return s -} diff --git a/internal/updater/zip.go b/internal/updater/zip.go deleted file mode 100644 index c3fb4fd2..00000000 --- a/internal/updater/zip.go +++ /dev/null @@ -1,78 +0,0 @@ -package updater - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "io/ioutil" - "net/http" - "path/filepath" - "strings" -) - -func fetchAndExtractFiles(ctx context.Context, client *http.Client, urls ...string) ( - contents map[string][]byte, err error) { - contents = make(map[string][]byte) - for _, url := range urls { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - response, err := client.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: %s for %s", ErrHTTPStatusCodeNotOK, response.Status, url) - } - - b, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - - if err := response.Body.Close(); err != nil { - return nil, err - } - - newContents, err := zipExtractAll(b) - if err != nil { - return nil, err - } - for fileName, content := range newContents { - contents[fileName] = content - } - } - return contents, nil -} - -func zipExtractAll(zipBytes []byte) (contents map[string][]byte, err error) { - r, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes))) - if err != nil { - return nil, err - } - contents = map[string][]byte{} - for _, zf := range r.File { - fileName := filepath.Base(zf.Name) - if !strings.HasSuffix(fileName, ".ovpn") { - continue - } - f, err := zf.Open() - if err != nil { - return nil, err - } - defer f.Close() - contents[fileName], err = ioutil.ReadAll(f) - if err != nil { - return nil, err - } - if err := f.Close(); err != nil { - return nil, err - } - } - return contents, nil -}