feat(publicip): resilient public ip fetcher (#2518)

- `PUBLICIP_API` accepts a comma separated list of ip data sources, where the first one is the base default one, and sources after it are backup sources used if we are rate limited.
- `PUBLICIP_API` defaults to `ipinfo,ifconfigco,ip2location,cloudflare` such that it now has `ifconfigco,ip2location,cloudflare` as backup ip data sources.
- `PUBLICIP_API_TOKEN` accepts a comma separated list of ip data source tokens, each corresponding by position to the APIs listed in `PUBLICIP_API`.
- logs ip data source when logging public ip information
- assume a rate limiting error is for 30 days (no persistence)
- ready for future live settings updates
  - consider an ip data source no longer banned if the token changes
  - keeps track of ban times when updating the list of fetchers

Dockerfile

@@ -205,7 +205,7 @@ ENV VPN_SERVICE_PROVIDER=pia \
     # Public IP
     PUBLICIP_FILE="/tmp/gluetun/ip" \
     PUBLICIP_ENABLED=on \
-    PUBLICIP_API=ipinfo \
+    PUBLICIP_API=ipinfo,ifconfigco,ip2location,cloudflare \
     PUBLICIP_API_TOKEN= \
     # Storage
     STORAGE_FILEPATH=/gluetun/servers.json \

cmd/gluetun/main.go

@@ -34,7 +34,6 @@ import (
 	"github.com/qdm12/gluetun/internal/pprof"
 	"github.com/qdm12/gluetun/internal/provider"
 	"github.com/qdm12/gluetun/internal/publicip"
-	pubipapi "github.com/qdm12/gluetun/internal/publicip/api"
 	"github.com/qdm12/gluetun/internal/routing"
 	"github.com/qdm12/gluetun/internal/server"
 	"github.com/qdm12/gluetun/internal/shadowsocks"
@@ -407,14 +406,11 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 	go dnsLooper.RunRestartTicker(dnsTickerCtx, dnsTickerDone)
 	controlGroupHandler.Add(dnsTickerHandler)
 
-	publicipAPI, _ := pubipapi.ParseProvider(allSettings.PublicIP.API)
-	ipFetcher, err := pubipapi.New(publicipAPI, httpClient, *allSettings.PublicIP.APIToken)
+	publicIPLooper, err := publicip.NewLoop(allSettings.PublicIP, puid, pgid, httpClient,
+		logger.New(log.SetComponent("ip getter")))
 	if err != nil {
-		return fmt.Errorf("creating public IP API client: %w", err)
+		return fmt.Errorf("creating public ip loop: %w", err)
 	}
-	publicIPLooper := publicip.NewLoop(ipFetcher,
-		logger.New(log.SetComponent("ip getter")),
-		allSettings.PublicIP, puid, pgid)
 	publicIPRunError, err := publicIPLooper.Start(ctx)
 	if err != nil {
 		return fmt.Errorf("starting public ip loop: %w", err)
@@ -426,7 +422,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 	parallelResolver := resolver.NewParallelResolver(allSettings.Updater.DNSAddress)
 	openvpnFileExtractor := extract.New()
 	providers := provider.NewProviders(storage, time.Now, updaterLogger,
-		httpClient, unzipper, parallelResolver, ipFetcher, openvpnFileExtractor)
+		httpClient, unzipper, parallelResolver, publicIPLooper.Fetcher(), openvpnFileExtractor)
 
 	vpnLogger := logger.New(log.SetComponent("vpn"))
 	vpnLooper := vpn.NewLoop(allSettings.VPN, ipv6Supported, allSettings.Firewall.VPNInputPorts,

internal/cli/update.go

@@ -80,10 +80,17 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 	httpClient := &http.Client{Timeout: clientTimeout}
 	unzipper := unzip.New(httpClient)
 	parallelResolver := resolver.NewParallelResolver(options.DNSAddress)
-	ipFetcher, err := api.New(api.IPInfo, httpClient, ipToken)
-	if err != nil {
-		return fmt.Errorf("creating public IP API client: %w", err)
+	nameTokenPairs := []api.NameToken{
+		{Name: string(api.IPInfo), Token: ipToken},
+		{Name: string(api.IP2Location)},
+		{Name: string(api.IfConfigCo)},
 	}
+	fetchers, err := api.New(nameTokenPairs, httpClient)
+	if err != nil {
+		return fmt.Errorf("creating public IP fetchers: %w", err)
+	}
+	ipFetcher := api.NewResilient(fetchers, logger)
+
 	openvpnFileExtractor := extract.New()
 
 	providers := provider.NewProviders(storage, time.Now, logger, httpClient,

internal/configuration/settings/helpers_test.go

@@ -1,3 +1,30 @@
 package settings
 
-func boolPtr(b bool) *bool { return &b }
+import gomock "github.com/golang/mock/gomock"
+
+type sourceKeyValue struct {
+	key   string
+	value string
+}
+
+func newMockSource(ctrl *gomock.Controller, keyValues []sourceKeyValue) *MockSource {
+	source := NewMockSource(ctrl)
+	var previousCall *gomock.Call
+	for _, keyValue := range keyValues {
+		transformedKey := keyValue.key
+		keyTransformCall := source.EXPECT().KeyTransform(keyValue.key).Return(transformedKey)
+		if previousCall != nil {
+			keyTransformCall.After(previousCall)
+		}
+		isSet := keyValue.value != ""
+		previousCall = source.EXPECT().Get(transformedKey).
+			Return(keyValue.value, isSet).After(keyTransformCall)
+		if isSet {
+			previousCall = source.EXPECT().KeyTransform(keyValue.key).
+				Return(transformedKey).After(previousCall)
+			previousCall = source.EXPECT().String().
+				Return("mock source").After(previousCall)
+		}
+	}
+	return source
+}

internal/configuration/settings/mocks_generate_test.go

@@ -0,0 +1,4 @@
+package settings
+
+//go:generate mockgen -destination=mocks_test.go -package=$GOPACKAGE . Warner
+//go:generate mockgen -destination=mocks_reader_test.go -package=$GOPACKAGE github.com/qdm12/gosettings/reader Source

internal/configuration/settings/mocks_reader_test.go

@@ -0,0 +1,77 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/qdm12/gosettings/reader (interfaces: Source)
+
+// Package settings is a generated GoMock package.
+package settings
+
+import (
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockSource is a mock of Source interface.
+type MockSource struct {
+	ctrl     *gomock.Controller
+	recorder *MockSourceMockRecorder
+}
+
+// MockSourceMockRecorder is the mock recorder for MockSource.
+type MockSourceMockRecorder struct {
+	mock *MockSource
+}
+
+// NewMockSource creates a new mock instance.
+func NewMockSource(ctrl *gomock.Controller) *MockSource {
+	mock := &MockSource{ctrl: ctrl}
+	mock.recorder = &MockSourceMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockSource) EXPECT() *MockSourceMockRecorder {
+	return m.recorder
+}
+
+// Get mocks base method.
+func (m *MockSource) Get(arg0 string) (string, bool) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(bool)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockSourceMockRecorder) Get(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockSource)(nil).Get), arg0)
+}
+
+// KeyTransform mocks base method.
+func (m *MockSource) KeyTransform(arg0 string) string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "KeyTransform", arg0)
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// KeyTransform indicates an expected call of KeyTransform.
+func (mr *MockSourceMockRecorder) KeyTransform(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyTransform", reflect.TypeOf((*MockSource)(nil).KeyTransform), arg0)
+}
+
+// String mocks base method.
+func (m *MockSource) String() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "String")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// String indicates an expected call of String.
+func (mr *MockSourceMockRecorder) String() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockSource)(nil).String))
+}

internal/configuration/settings/mocks_test.go

@@ -0,0 +1,46 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/qdm12/gluetun/internal/configuration/settings (interfaces: Warner)
+
+// Package settings is a generated GoMock package.
+package settings
+
+import (
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockWarner is a mock of Warner interface.
+type MockWarner struct {
+	ctrl     *gomock.Controller
+	recorder *MockWarnerMockRecorder
+}
+
+// MockWarnerMockRecorder is the mock recorder for MockWarner.
+type MockWarnerMockRecorder struct {
+	mock *MockWarner
+}
+
+// NewMockWarner creates a new mock instance.
+func NewMockWarner(ctrl *gomock.Controller) *MockWarner {
+	mock := &MockWarner{ctrl: ctrl}
+	mock.recorder = &MockWarnerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockWarner) EXPECT() *MockWarnerMockRecorder {
+	return m.recorder
+}
+
+// Warn mocks base method.
+func (m *MockWarner) Warn(arg0 string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Warn", arg0)
+}
+
+// Warn indicates an expected call of Warn.
+func (mr *MockWarnerMockRecorder) Warn(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Warn", reflect.TypeOf((*MockWarner)(nil).Warn), arg0)
+}

internal/configuration/settings/portforward_test.go

@@ -10,7 +10,7 @@ func Test_PortForwarding_String(t *testing.T) {
 	t.Parallel()
 
 	settings := PortForwarding{
-		Enabled: boolPtr(false),
+		Enabled: ptrTo(false),
 	}
 
 	s := settings.String()

internal/configuration/settings/publicip.go

@@ -20,15 +20,20 @@ type PublicIP struct {
 	// to write to a file. It cannot be nil for the
 	// internal state
 	IPFilepath *string
-	// API is the API name to use to fetch public IP information.
-	// It can be cloudflare, ifconfigco, ip2location or ipinfo.
-	// It defaults to ipinfo.
-	API string
-	// APIToken is the token to use for the IP data service
-	// such as ipinfo.io. It can be the empty string to
-	// indicate not to use a token. It cannot be nil for the
-	// internal state.
-	APIToken *string
+	// APIs is the list of public ip APIs to use to fetch public IP information.
+	// If there is more than one API, the first one is used
+	// by default and the others are used as fallbacks in case of
+	// the service rate limiting us. It defaults to use all services,
+	// with the first one being ipinfo.io for historical reasons.
+	APIs []PublicIPAPI
+}
+
+type PublicIPAPI struct {
+	// Name is the name of the public ip API service.
+	// It can be "cloudflare", "ifconfigco", "ip2location" or "ipinfo".
+	Name string
+	// Token is the token to use for the public ip API service.
+	Token string
 }
 
 // UpdateWith deep copies the receiving settings, overrides the copy with
@@ -53,9 +58,11 @@ func (p PublicIP) validate() (err error) {
 		}
 	}
 
-	_, err = api.ParseProvider(p.API)
-	if err != nil {
-		return fmt.Errorf("API name: %w", err)
+	for _, publicIPAPI := range p.APIs {
+		_, err = api.ParseProvider(publicIPAPI.Name)
+		if err != nil {
+			return fmt.Errorf("API name: %w", err)
+		}
 	}
 
 	return nil
@@ -65,23 +72,25 @@ func (p *PublicIP) copy() (copied PublicIP) {
 	return PublicIP{
 		Enabled:    gosettings.CopyPointer(p.Enabled),
 		IPFilepath: gosettings.CopyPointer(p.IPFilepath),
-		API:        p.API,
-		APIToken:   gosettings.CopyPointer(p.APIToken),
+		APIs:       gosettings.CopySlice(p.APIs),
 	}
 }
 
 func (p *PublicIP) overrideWith(other PublicIP) {
 	p.Enabled = gosettings.OverrideWithPointer(p.Enabled, other.Enabled)
 	p.IPFilepath = gosettings.OverrideWithPointer(p.IPFilepath, other.IPFilepath)
-	p.API = gosettings.OverrideWithComparable(p.API, other.API)
-	p.APIToken = gosettings.OverrideWithPointer(p.APIToken, other.APIToken)
+	p.APIs = gosettings.OverrideWithSlice(p.APIs, other.APIs)
 }
 
 func (p *PublicIP) setDefaults() {
 	p.Enabled = gosettings.DefaultPointer(p.Enabled, true)
 	p.IPFilepath = gosettings.DefaultPointer(p.IPFilepath, "/tmp/gluetun/ip")
-	p.API = gosettings.DefaultComparable(p.API, "ipinfo")
-	p.APIToken = gosettings.DefaultPointer(p.APIToken, "")
+	p.APIs = gosettings.DefaultSlice(p.APIs, []PublicIPAPI{
+		{Name: string(api.IPInfo)},
+		{Name: string(api.Cloudflare)},
+		{Name: string(api.IfConfigCo)},
+		{Name: string(api.IP2Location)},
+	})
 }
 
 func (p PublicIP) String() string {
@@ -99,10 +108,20 @@ func (p PublicIP) toLinesNode() (node *gotree.Node) {
 		node.Appendf("IP file path: %s", *p.IPFilepath)
 	}
 
-	node.Appendf("Public IP data API: %s", p.API)
-
-	if *p.APIToken != "" {
-		node.Appendf("API token: %s", gosettings.ObfuscateKey(*p.APIToken))
+	baseAPIString := "Public IP data base API: " + p.APIs[0].Name
+	if p.APIs[0].Token != "" {
+		baseAPIString += " (token " + gosettings.ObfuscateKey(p.APIs[0].Token) + ")"
+	}
+	node.Append(baseAPIString)
+	if len(p.APIs) > 1 {
+		backupAPIsNode := node.Append("Public IP data backup APIs:")
+		for i := 1; i < len(p.APIs); i++ {
+			message := p.APIs[i].Name
+			if p.APIs[i].Token != "" {
+				message += " (token " + gosettings.ObfuscateKey(p.APIs[i].Token) + ")"
+			}
+			backupAPIsNode.Append(message)
+		}
 	}
 
 	return node
@@ -116,8 +135,21 @@ func (p *PublicIP) read(r *reader.Reader, warner Warner) (err error) {
 
 	p.IPFilepath = r.Get("PUBLICIP_FILE",
 		reader.ForceLowercase(false), reader.RetroKeys("IP_STATUS_FILE"))
-	p.API = r.String("PUBLICIP_API")
-	p.APIToken = r.Get("PUBLICIP_API_TOKEN")
+
+	apiNames := r.CSV("PUBLICIP_API")
+	if len(apiNames) > 0 {
+		apiTokens := r.CSV("PUBLICIP_API_TOKEN")
+		p.APIs = make([]PublicIPAPI, len(apiNames))
+		for i := range apiNames {
+			p.APIs[i].Name = apiNames[i]
+			var token string
+			if i < len(apiTokens) { // only set token if it exists
+				token = apiTokens[i]
+			}
+			p.APIs[i].Token = token
+		}
+	}
+
 	return nil
 }
 

internal/configuration/settings/publicip_test.go

@@ -0,0 +1,161 @@
+package settings
+
+import (
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/qdm12/gosettings/reader"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_PublicIP_read(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		makeReader func(ctrl *gomock.Controller) *reader.Reader
+		makeWarner func(ctrl *gomock.Controller) Warner
+		settings   PublicIP
+		errWrapped error
+		errMessage string
+	}{
+		"nothing_read": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+		},
+		"single_api_no_token": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API", value: "ipinfo"},
+					{key: "PUBLICIP_API_TOKEN"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+			settings: PublicIP{
+				APIs: []PublicIPAPI{
+					{Name: "ipinfo"},
+				},
+			},
+		},
+		"single_api_with_token": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API", value: "ipinfo"},
+					{key: "PUBLICIP_API_TOKEN", value: "xyz"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+			settings: PublicIP{
+				APIs: []PublicIPAPI{
+					{Name: "ipinfo", Token: "xyz"},
+				},
+			},
+		},
+		"multiple_apis_no_token": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API", value: "ipinfo,ip2location"},
+					{key: "PUBLICIP_API_TOKEN"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+			settings: PublicIP{
+				APIs: []PublicIPAPI{
+					{Name: "ipinfo"},
+					{Name: "ip2location"},
+				},
+			},
+		},
+		"multiple_apis_with_token": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API", value: "ipinfo,ip2location"},
+					{key: "PUBLICIP_API_TOKEN", value: "xyz,abc"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+			settings: PublicIP{
+				APIs: []PublicIPAPI{
+					{Name: "ipinfo", Token: "xyz"},
+					{Name: "ip2location", Token: "abc"},
+				},
+			},
+		},
+		"multiple_apis_with_and_without_token": {
+			makeReader: func(ctrl *gomock.Controller) *reader.Reader {
+				source := newMockSource(ctrl, []sourceKeyValue{
+					{key: "PUBLICIP_PERIOD"},
+					{key: "PUBLICIP_ENABLED"},
+					{key: "IP_STATUS_FILE"},
+					{key: "PUBLICIP_FILE"},
+					{key: "PUBLICIP_API", value: "ipinfo,ip2location"},
+					{key: "PUBLICIP_API_TOKEN", value: "xyz"},
+				})
+				return reader.New(reader.Settings{
+					Sources: []reader.Source{source},
+				})
+			},
+			settings: PublicIP{
+				APIs: []PublicIPAPI{
+					{Name: "ipinfo", Token: "xyz"},
+					{Name: "ip2location"},
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			ctrl := gomock.NewController(t)
+
+			reader := testCase.makeReader(ctrl)
+			var warner Warner
+			if testCase.makeWarner != nil {
+				warner = testCase.makeWarner(ctrl)
+			}
+
+			var settings PublicIP
+			err := settings.read(reader, warner)
+
+			assert.Equal(t, testCase.settings, settings)
+			assert.ErrorIs(t, err, testCase.errWrapped)
+			if testCase.errWrapped != nil {
+				assert.EqualError(t, err, testCase.errMessage)
+			}
+		})
+	}
+}

internal/configuration/settings/settings_test.go

@@ -79,7 +79,11 @@ func Test_Settings_String(t *testing.T) {
 |   └── Process GID: 1000
 ├── Public IP settings:
 |   ├── IP file path: /tmp/gluetun/ip
-|   └── Public IP data API: ipinfo
+|   ├── Public IP data base API: ipinfo
+|   └── Public IP data backup APIs:
+|       ├── cloudflare
+|       ├── ifconfigco
+|       └── ip2location
 └── Version settings:
     └── Enabled: yes`,
 		},

internal/publicip/api/api.go

@@ -1,23 +1,12 @@
 package api
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/http"
-	"net/netip"
 	"strings"
-
-	"github.com/qdm12/gluetun/internal/models"
 )
 
-type API interface {
-	String() string
-	CanFetchAnyIP() bool
-	FetchInfo(ctx context.Context, ip netip.Addr) (
-		result models.PublicIP, err error)
-}
-
 type Provider string
 
 const (
@@ -27,21 +16,34 @@ const (
 	IP2Location Provider = "ip2location"
 )
 
-func New(provider Provider, client *http.Client, token string) ( //nolint:ireturn
-	a API, err error,
+type NameToken struct {
+	Name  string
+	Token string
+}
+
+func New(nameTokenPairs []NameToken, client *http.Client) (
+	fetchers []Fetcher, err error,
 ) {
-	switch provider {
-	case Cloudflare:
-		return newCloudflare(client), nil
-	case IfConfigCo:
-		return newIfConfigCo(client), nil
-	case IPInfo:
-		return newIPInfo(client, token), nil
-	case IP2Location:
-		return newIP2Location(client, token), nil
-	default:
-		panic("provider not valid: " + provider)
+	fetchers = make([]Fetcher, len(nameTokenPairs))
+	for i, nameTokenPair := range nameTokenPairs {
+		provider, err := ParseProvider(nameTokenPair.Name)
+		if err != nil {
+			return nil, fmt.Errorf("parsing API name: %w", err)
+		}
+		switch provider {
+		case Cloudflare:
+			fetchers[i] = newCloudflare(client)
+		case IfConfigCo:
+			fetchers[i] = newIfConfigCo(client)
+		case IPInfo:
+			fetchers[i] = newIPInfo(client, nameTokenPair.Token)
+		case IP2Location:
+			fetchers[i] = newIP2Location(client, nameTokenPair.Token)
+		default:
+			panic("provider not valid: " + provider)
+		}
 	}
+	return fetchers, nil
 }
 
 var ErrProviderNotValid = errors.New("API name is not valid")

internal/publicip/api/cloudflare.go

@@ -30,6 +30,10 @@ func (c *cloudflare) CanFetchAnyIP() bool {
 	return false
 }
 
+func (c *cloudflare) Token() (token string) {
+	return ""
+}
+
 // FetchInfo obtains information on the public IP address of the machine,
 // and returns an error if the `ip` argument is set since the Cloudflare API
 // can only be used to provide details about the current machine public IP.

internal/publicip/api/ifconfigco.go

@@ -28,6 +28,10 @@ func (i *ifConfigCo) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ifConfigCo) Token() string {
+	return ""
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the ifconfig.co/json API. If the ip is the zero value,
 // the public IP address of the machine is used as the IP.

internal/publicip/api/interfaces.go

@@ -8,6 +8,17 @@ import (
 )
 
 type Fetcher interface {
+	String() string
+	CanFetchAnyIP() bool
+	Token() (token string)
+	InfoFetcher
+}
+
+type InfoFetcher interface {
 	FetchInfo(ctx context.Context, ip netip.Addr) (
 		result models.PublicIP, err error)
 }
+
+type Warner interface {
+	Warn(message string)
+}

internal/publicip/api/ip2location.go

@@ -31,6 +31,10 @@ func (i *ip2Location) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ip2Location) Token() string {
+	return i.token
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the api.ip2location.io API. If the ip is the zero value,
 // the public IP address of the machine is used as the IP.

internal/publicip/api/ipinfo.go

@@ -32,6 +32,10 @@ func (i *ipInfo) CanFetchAnyIP() bool {
 	return true
 }
 
+func (i *ipInfo) Token() string {
+	return i.token
+}
+
 // FetchInfo obtains information on the ip address provided
 // using the ipinfo.io API. If the ip is the zero value, the public IP address
 // of the machine is used as the IP.

internal/publicip/api/multi.go

@@ -13,7 +13,7 @@ import (
 // If an error is encountered, all the operations are canceled and
 // an error is returned, so the results returned should be considered
 // incomplete in this case.
-func FetchMultiInfo(ctx context.Context, fetcher Fetcher, ips []netip.Addr) (
+func FetchMultiInfo(ctx context.Context, fetcher InfoFetcher, ips []netip.Addr) (
 	results []models.PublicIP, err error,
 ) {
 	ctx, cancel := context.WithCancel(ctx)

internal/publicip/api/resilient.go

@@ -0,0 +1,153 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/netip"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/qdm12/gluetun/internal/models"
+)
+
+type ResilientFetcher struct {
+	fetchers         []Fetcher
+	logger           Warner
+	fetcherToBanTime map[Fetcher]time.Time
+	mutex            sync.RWMutex
+	timeNow          func() time.Time
+}
+
+// NewResilient creates a 'resilient' fetcher given multiple fetchers.
+// For example, it can handle bans and move on to another fetcher if one fails.
+func NewResilient(fetchers []Fetcher, logger Warner) *ResilientFetcher {
+	return &ResilientFetcher{
+		fetchers:         fetchers,
+		logger:           logger,
+		fetcherToBanTime: make(map[Fetcher]time.Time, len(fetchers)),
+		timeNow:          time.Now,
+	}
+}
+
+func (r *ResilientFetcher) isBanned(fetcher Fetcher) (banned bool) {
+	banTime, banned := r.fetcherToBanTime[fetcher]
+	if !banned {
+		return false
+	}
+	const banDuration = 30 * 24 * time.Hour
+	banExpiryTime := banTime.Add(banDuration)
+	now := r.timeNow()
+	if now.After(banExpiryTime) {
+		delete(r.fetcherToBanTime, fetcher)
+		return false
+	}
+	return true
+}
+
+func (r *ResilientFetcher) String() string {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) {
+			continue
+		}
+		return fetcher.String()
+	}
+	return "<all-banned>"
+}
+
+func (r *ResilientFetcher) Token() string {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) {
+			continue
+		}
+		return fetcher.Token()
+	}
+	return "<all-banned>"
+}
+
+// CanFetchAnyIP returns true if any of the fetchers
+// can fetch any IP address and is not banned.
+func (r *ResilientFetcher) CanFetchAnyIP() bool {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+
+	for _, fetcher := range r.fetchers {
+		if !fetcher.CanFetchAnyIP() || r.isBanned(fetcher) {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+var ErrFetchersAllRateLimited = errors.New("all fetchers are rate limited")
+
+// FetchInfo obtains information on the ip address provided.
+// If the ip is the zero value, the public IP address of the machine
+// is used as the IP.
+// If a fetcher gets banned, the next one is tried – until all have been exhausted.
+// Fetchers still within their banned period are skipped.
+// If an error unrelated to being banned is encountered, it is returned and more
+// fetchers are tried.
+func (r *ResilientFetcher) FetchInfo(ctx context.Context, ip netip.Addr) (
+	result models.PublicIP, err error,
+) {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+
+	for _, fetcher := range r.fetchers {
+		if r.isBanned(fetcher) ||
+			(ip.IsValid() && !fetcher.CanFetchAnyIP()) {
+			continue
+		}
+
+		result, err = fetcher.FetchInfo(ctx, ip)
+		if err == nil || !errors.Is(err, ErrTooManyRequests) {
+			return result, err
+		}
+
+		// Fetcher is banned
+		r.fetcherToBanTime[fetcher] = r.timeNow()
+		r.logger.Warn(fetcher.String() + ": " + err.Error())
+	}
+
+	fetcherNames := make([]string, len(r.fetchers))
+	for i, fetcher := range r.fetchers {
+		fetcherNames[i] = fetcher.String()
+	}
+
+	return result, fmt.Errorf("%w (%s)",
+		ErrFetchersAllRateLimited,
+		strings.Join(fetcherNames, ", "))
+}
+
+func (r *ResilientFetcher) UpdateFetchers(fetchers []Fetcher) {
+	newFetcherNameToFetcher := make(map[string]Fetcher, len(fetchers))
+	for _, fetcher := range fetchers {
+		newFetcherNameToFetcher[fetcher.String()] = fetcher
+	}
+
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	newFetcherToBanTime := make(map[Fetcher]time.Time, len(r.fetcherToBanTime))
+	for bannedFetcher, banTime := range r.fetcherToBanTime {
+		if !r.isBanned(bannedFetcher) {
+			// fetcher is no longer in its ban period.
+			continue
+		}
+		bannedName := bannedFetcher.String()
+		newFetcher, isNewFetcher := newFetcherNameToFetcher[bannedName]
+		if isNewFetcher && newFetcher.Token() == bannedFetcher.Token() {
+			newFetcherToBanTime[newFetcher] = banTime
+		}
+	}
+
+	r.fetchers = fetchers
+	r.fetcherToBanTime = newFetcherToBanTime
+}

internal/publicip/interfaces.go

@@ -1,18 +1,5 @@
 package publicip
 
-import (
-	"context"
-	"net/netip"
-
-	"github.com/qdm12/gluetun/internal/models"
-)
-
-type Fetcher interface {
-	String() string
-	FetchInfo(ctx context.Context, ip netip.Addr) (
-		result models.PublicIP, err error)
-}
-
 type Logger interface {
 	Info(s string)
 	Warn(s string)

internal/publicip/loop.go

@@ -3,12 +3,14 @@ package publicip
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/netip"
 	"sync"
 	"time"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/models"
+	"github.com/qdm12/gluetun/internal/publicip/api"
 )
 
 type Loop struct {
@@ -17,9 +19,10 @@ type Loop struct {
 	settingsMutex sync.RWMutex
 	ipData        models.PublicIP
 	ipDataMutex   sync.RWMutex
+	fetcher       *api.ResilientFetcher
 	// Fixed injected objects
-	fetcher Fetcher
-	logger  Logger
+	httpClient *http.Client
+	logger     Logger
 	// Fixed parameters
 	puid int
 	pgid int
@@ -37,17 +40,34 @@ type Loop struct {
 	timeNow func() time.Time
 }
 
-func NewLoop(fetcher Fetcher, logger Logger,
-	settings settings.PublicIP, puid, pgid int,
-) *Loop {
-	return &Loop{
-		settings: settings,
-		fetcher:  fetcher,
-		logger:   logger,
-		puid:     puid,
-		pgid:     pgid,
-		timeNow:  time.Now,
+func NewLoop(settings settings.PublicIP, puid, pgid int,
+	httpClient *http.Client, logger Logger,
+) (loop *Loop, err error) {
+	fetchers, err := api.New(makeNameTokenPairs(settings.APIs), httpClient)
+	if err != nil {
+		return nil, fmt.Errorf("creating fetchers: %w", err)
 	}
+
+	return &Loop{
+		settings:   settings,
+		httpClient: httpClient,
+		fetcher:    api.NewResilient(fetchers, logger),
+		logger:     logger,
+		puid:       puid,
+		pgid:       pgid,
+		timeNow:    time.Now,
+	}, nil
+}
+
+func makeNameTokenPairs(apis []settings.PublicIPAPI) (nameTokenPairs []api.NameToken) {
+	nameTokenPairs = make([]api.NameToken, len(apis))
+	for i := range apis {
+		nameTokenPairs[i] = api.NameToken{
+			Name:  apis[i].Name,
+			Token: apis[i].Token,
+		}
+	}
+	return nameTokenPairs
 }
 
 func (l *Loop) String() string {
@@ -102,7 +122,8 @@ func (l *Loop) run(runCtx context.Context, runDone chan<- struct{},
 		}
 
 		message := "Public IP address is " + result.IP.String()
-		message += " (" + result.Country + ", " + result.Region + ", " + result.City + ")"
+		message += " (" + result.Country + ", " + result.Region + ", " + result.City +
+			" - source: " + l.fetcher.String() + ")"
 		l.logger.Info(message)
 
 		l.ipDataMutex.Lock()
@@ -159,3 +180,7 @@ func (l *Loop) Stop() (err error) {
 	<-l.runDone
 	return l.ClearData()
 }
+
+func (l *Loop) Fetcher() (fetcher *api.ResilientFetcher) {
+	return l.fetcher
+}

internal/publicip/update.go

@@ -3,8 +3,10 @@ package publicip
 import (
 	"fmt"
 	"os"
+	"reflect"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/publicip/api"
 )
 
 func (l *Loop) update(partialUpdate settings.PublicIP) (err error) {
@@ -37,6 +39,15 @@ func (l *Loop) update(partialUpdate settings.PublicIP) (err error) {
 		}
 	}
 
+	if !reflect.DeepEqual(l.settings.APIs, updatedSettings.APIs) {
+		newFetchers, err := api.New(makeNameTokenPairs(updatedSettings.APIs), l.httpClient)
+		if err != nil {
+			return fmt.Errorf("creating fetchers: %w", err)
+		}
+
+		l.fetcher.UpdateFetchers(newFetchers)
+	}
+
 	l.settings = updatedSettings
 
 	return nil