feat(publicip): resilient public ip fetcher (#2518)

- `PUBLICIP_API` accepts a comma separated list of ip data sources, where the first one is the base default one, and sources after it are backup sources used if we are rate limited.
- `PUBLICIP_API` defaults to `ipinfo,ifconfigco,ip2location,cloudflare` such that it now has `ifconfigco,ip2location,cloudflare` as backup ip data sources.
- `PUBLICIP_API_TOKEN` accepts a comma separated list of ip data source tokens, each corresponding by position to the APIs listed in `PUBLICIP_API`.
- logs ip data source when logging public ip information
- assume a rate limiting error is for 30 days (no persistence)
- ready for future live settings updates
  - consider an ip data source no longer banned if the token changes
  - keeps track of ban times when updating the list of fetchers

internal/publicip/api/api.go

@@ -1,23 +1,12 @@
 package api
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/http"
-	"net/netip"
 	"strings"
-
-	"github.com/qdm12/gluetun/internal/models"
 )
 
-type API interface {
-	String() string
-	CanFetchAnyIP() bool
-	FetchInfo(ctx context.Context, ip netip.Addr) (
-		result models.PublicIP, err error)
-}
-
 type Provider string
 
 const (
@@ -27,21 +16,34 @@ const (
 	IP2Location Provider = "ip2location"
 )
 
-func New(provider Provider, client *http.Client, token string) ( //nolint:ireturn
-	a API, err error,
+type NameToken struct {
+	Name  string
+	Token string
+}
+
+func New(nameTokenPairs []NameToken, client *http.Client) (
+	fetchers []Fetcher, err error,
 ) {
-	switch provider {
-	case Cloudflare:
-		return newCloudflare(client), nil
-	case IfConfigCo:
-		return newIfConfigCo(client), nil
-	case IPInfo:
-		return newIPInfo(client, token), nil
-	case IP2Location:
-		return newIP2Location(client, token), nil
-	default:
-		panic("provider not valid: " + provider)
+	fetchers = make([]Fetcher, len(nameTokenPairs))
+	for i, nameTokenPair := range nameTokenPairs {
+		provider, err := ParseProvider(nameTokenPair.Name)
+		if err != nil {
+			return nil, fmt.Errorf("parsing API name: %w", err)
+		}
+		switch provider {
+		case Cloudflare:
+			fetchers[i] = newCloudflare(client)
+		case IfConfigCo:
+			fetchers[i] = newIfConfigCo(client)
+		case IPInfo:
+			fetchers[i] = newIPInfo(client, nameTokenPair.Token)
+		case IP2Location:
+			fetchers[i] = newIP2Location(client, nameTokenPair.Token)
+		default:
+			panic("provider not valid: " + provider)
+		}
 	}
+	return fetchers, nil
 }
 
 var ErrProviderNotValid = errors.New("API name is not valid")

internal/publicip/api/cloudflare.go

@@ -30,6 +30,10 @@ func (c *cloudflare) CanFetchAnyIP() bool {
 	return false
 }
 
+func (c *cloudflare) Token() (token string) {
+	return ""
+}
+
 // FetchInfo obtains information on the public IP address of the machine,
 // and returns an error if the `ip` argument is set since the Cloudflare API
 // can only be used to provide details about the current machine public IP.

internal/publicip/api/ifconfigco.go

@@ -28,6 +28,10 @@ func (i *ifConfigCo) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ifConfigCo) Token() string {
+	return ""
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the ifconfig.co/json API. If the ip is the zero value,
 // the public IP address of the machine is used as the IP.

internal/publicip/api/interfaces.go

@@ -8,6 +8,17 @@ import (
 )
 
 type Fetcher interface {
+	String() string
+	CanFetchAnyIP() bool
+	Token() (token string)
+	InfoFetcher
+}
+
+type InfoFetcher interface {
 	FetchInfo(ctx context.Context, ip netip.Addr) (
 		result models.PublicIP, err error)
 }
+
+type Warner interface {
+	Warn(message string)
+}

internal/publicip/api/ip2location.go

@@ -31,6 +31,10 @@ func (i *ip2Location) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ip2Location) Token() string {
+	return i.token
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the api.ip2location.io API. If the ip is the zero value,
 // the public IP address of the machine is used as the IP.

internal/publicip/api/ipinfo.go

@@ -32,6 +32,10 @@ func (i *ipInfo) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ipInfo) Token() string {
+	return i.token
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the ipinfo.io API. If the ip is the zero value, the public IP address
 // of the machine is used as the IP.

internal/publicip/api/multi.go

@@ -13,7 +13,7 @@ import (
 // If an error is encountered, all the operations are canceled and
 // an error is returned, so the results returned should be considered
 // incomplete in this case.
-func FetchMultiInfo(ctx context.Context, fetcher Fetcher, ips []netip.Addr) (
+func FetchMultiInfo(ctx context.Context, fetcher InfoFetcher, ips []netip.Addr) (
 	results []models.PublicIP, err error,
 ) {
 	ctx, cancel := context.WithCancel(ctx)

internal/publicip/api/resilient.go

@@ -0,0 +1,153 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/netip"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+type ResilientFetcher struct {
+	fetchers         []Fetcher
+	logger           Warner
+	fetcherToBanTime map[Fetcher]time.Time
+	mutex            sync.RWMutex
+	timeNow          func() time.Time
+}
+
+// NewResilient creates a 'resilient' fetcher given multiple fetchers.
+// For example, it can handle bans and move on to another fetcher if one fails.
+func NewResilient(fetchers []Fetcher, logger Warner) *ResilientFetcher {
+	return &ResilientFetcher{
+		fetchers:         fetchers,
+		logger:           logger,
+		fetcherToBanTime: make(map[Fetcher]time.Time, len(fetchers)),
+		timeNow:          time.Now,
+	}
+}
+
+func (r *ResilientFetcher) isBanned(fetcher Fetcher) (banned bool) {
+	banTime, banned := r.fetcherToBanTime[fetcher]
+	if !banned {
+		return false
+	}
+	const banDuration = 30 * 24 * time.Hour
+	banExpiryTime := banTime.Add(banDuration)
+	now := r.timeNow()
+	if now.After(banExpiryTime) {
+		delete(r.fetcherToBanTime, fetcher)
+		return false
+	}
+	return true
+}
+
+func (r *ResilientFetcher) String() string {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) {
+			continue
+		}
+		return fetcher.String()
+	}
+	return "<all-banned>"
+}
+
+func (r *ResilientFetcher) Token() string {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) {
+			continue
+		}
+		return fetcher.Token()
+	}
+	return "<all-banned>"
+}
+
+// CanFetchAnyIP returns true if any of the fetchers
+// can fetch any IP address and is not banned.
+func (r *ResilientFetcher) CanFetchAnyIP() bool {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+
+	for _, fetcher := range r.fetchers {
+		if !fetcher.CanFetchAnyIP() || r.isBanned(fetcher) {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+var ErrFetchersAllRateLimited = errors.New("all fetchers are rate limited")
+
+// FetchInfo obtains information on the ip address provided.
+// If the ip is the zero value, the public IP address of the machine
+// is used as the IP.
+// If a fetcher gets banned, the next one is tried – until all have been exhausted.
+// Fetchers still within their banned period are skipped.
+// If an error unrelated to being banned is encountered, it is returned and more
+// fetchers are tried.
+func (r *ResilientFetcher) FetchInfo(ctx context.Context, ip netip.Addr) (
+	result models.PublicIP, err error,
+) {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) ||
+			(ip.IsValid() && !fetcher.CanFetchAnyIP()) {
+			continue
+		}
+
+		result, err = fetcher.FetchInfo(ctx, ip)
+		if err == nil || !errors.Is(err, ErrTooManyRequests) {
+			return result, err
+		}
+
+		// Fetcher is banned
+		r.fetcherToBanTime[fetcher] = r.timeNow()
+		r.logger.Warn(fetcher.String() + ": " + err.Error())
+	}
+
+	fetcherNames := make([]string, len(r.fetchers))
+	for i, fetcher := range r.fetchers {
+		fetcherNames[i] = fetcher.String()
+	}
+
+	return result, fmt.Errorf("%w (%s)",
+		ErrFetchersAllRateLimited,
+		strings.Join(fetcherNames, ", "))
+}
+
+func (r *ResilientFetcher) UpdateFetchers(fetchers []Fetcher) {
+	newFetcherNameToFetcher := make(map[string]Fetcher, len(fetchers))
+	for _, fetcher := range fetchers {
+		newFetcherNameToFetcher[fetcher.String()] = fetcher
+	}
+
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	newFetcherToBanTime := make(map[Fetcher]time.Time, len(r.fetcherToBanTime))
+	for bannedFetcher, banTime := range r.fetcherToBanTime {
+		if !r.isBanned(bannedFetcher) {
+			// fetcher is no longer in its ban period.
+			continue
+		}
+		bannedName := bannedFetcher.String()
+		newFetcher, isNewFetcher := newFetcherNameToFetcher[bannedName]
+		if isNewFetcher && newFetcher.Token() == bannedFetcher.Token() {
+			newFetcherToBanTime[newFetcher] = banTime
+		}
+	}
+
+	r.fetchers = fetchers
+	r.fetcherToBanTime = newFetcherToBanTime
+}

internal/publicip/interfaces.go

@@ -1,18 +1,5 @@
 package publicip
 
-import (
-	"context"
-	"net/netip"
-
-	"github.com/qdm12/gluetun/internal/models"
-)
-
-type Fetcher interface {
-	String() string
-	FetchInfo(ctx context.Context, ip netip.Addr) (
-		result models.PublicIP, err error)
-}
-
 type Logger interface {
 	Info(s string)
 	Warn(s string)

internal/publicip/loop.go

@@ -3,12 +3,14 @@ package publicip
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/netip"
 	"sync"
 	"time"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/publicip/api"
 )
 
 type Loop struct {
@@ -17,9 +19,10 @@ type Loop struct {
 	settingsMutex sync.RWMutex
 	ipData        models.PublicIP
 	ipDataMutex   sync.RWMutex
+	fetcher       *api.ResilientFetcher
 	// Fixed injected objects
-	fetcher Fetcher
-	logger  Logger
+	httpClient *http.Client
+	logger     Logger
 	// Fixed parameters
 	puid int
 	pgid int
@@ -37,17 +40,34 @@ type Loop struct {
 	timeNow func() time.Time
 }
 
-func NewLoop(fetcher Fetcher, logger Logger,
-	settings settings.PublicIP, puid, pgid int,
-) *Loop {
-	return &Loop{
-		settings: settings,
-		fetcher:  fetcher,
-		logger:   logger,
-		puid:     puid,
-		pgid:     pgid,
-		timeNow:  time.Now,
+func NewLoop(settings settings.PublicIP, puid, pgid int,
+	httpClient *http.Client, logger Logger,
+) (loop *Loop, err error) {
+	fetchers, err := api.New(makeNameTokenPairs(settings.APIs), httpClient)
+	if err != nil {
+		return nil, fmt.Errorf("creating fetchers: %w", err)
 	}
+
+	return &Loop{
+		settings:   settings,
+		httpClient: httpClient,
+		fetcher:    api.NewResilient(fetchers, logger),
+		logger:     logger,
+		puid:       puid,
+		pgid:       pgid,
+		timeNow:    time.Now,
+	}, nil
+}
+
+func makeNameTokenPairs(apis []settings.PublicIPAPI) (nameTokenPairs []api.NameToken) {
+	nameTokenPairs = make([]api.NameToken, len(apis))
+	for i := range apis {
+		nameTokenPairs[i] = api.NameToken{
+			Name:  apis[i].Name,
+			Token: apis[i].Token,
+		}
+	}
+	return nameTokenPairs
 }
 
 func (l *Loop) String() string {
@@ -102,7 +122,8 @@ func (l *Loop) run(runCtx context.Context, runDone chan<- struct{},
 		}
 
 		message := "Public IP address is " + result.IP.String()
-		message += " (" + result.Country + ", " + result.Region + ", " + result.City + ")"
+		message += " (" + result.Country + ", " + result.Region + ", " + result.City +
+			" - source: " + l.fetcher.String() + ")"
 		l.logger.Info(message)
 
 		l.ipDataMutex.Lock()
@@ -159,3 +180,7 @@ func (l *Loop) Stop() (err error) {
 	<-l.runDone
 	return l.ClearData()
 }
+
+func (l *Loop) Fetcher() (fetcher *api.ResilientFetcher) {
+	return l.fetcher
+}

internal/publicip/update.go

@@ -3,8 +3,10 @@ package publicip
 import (
 	"fmt"
 	"os"
+	"reflect"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/publicip/api"
 )
 
 func (l *Loop) update(partialUpdate settings.PublicIP) (err error) {
@@ -37,6 +39,15 @@ func (l *Loop) update(partialUpdate settings.PublicIP) (err error) {
 		}
 	}
 
+	if !reflect.DeepEqual(l.settings.APIs, updatedSettings.APIs) {
+		newFetchers, err := api.New(makeNameTokenPairs(updatedSettings.APIs), l.httpClient)
+		if err != nil {
+			return fmt.Errorf("creating fetchers: %w", err)
+		}
+
+		l.fetcher.UpdateFetchers(newFetchers)
+	}
+
 	l.settings = updatedSettings
 
 	return nil