From a61302f1359fb69b6bf3157c98521252773046bb Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Sat, 19 Oct 2024 15:21:14 +0200 Subject: [PATCH] feat(publicip): resilient public ip fetcher (#2518) - `PUBLICIP_API` accepts a comma separated list of ip data sources, where the first one is the base default one, and sources after it are backup sources used if we are rate limited. - `PUBLICIP_API` defaults to `ipinfo,ifconfigco,ip2location,cloudflare` such that it now has `ifconfigco,ip2location,cloudflare` as backup ip data sources. - `PUBLICIP_API_TOKEN` accepts a comma separated list of ip data source tokens, each corresponding by position to the APIs listed in `PUBLICIP_API`. - logs ip data source when logging public ip information - assume a rate limiting error is for 30 days (no persistence) - ready for future live settings updates - consider an ip data source no longer banned if the token changes - keeps track of ban times when updating the list of fetchers --- Dockerfile | 2 +- cmd/gluetun/main.go | 12 +- internal/cli/update.go | 13 +- .../configuration/settings/helpers_test.go | 29 +++- .../settings/mocks_generate_test.go | 4 + .../settings/mocks_reader_test.go | 77 +++++++++ internal/configuration/settings/mocks_test.go | 46 +++++ .../settings/portforward_test.go | 2 +- internal/configuration/settings/publicip.go | 80 ++++++--- .../configuration/settings/publicip_test.go | 161 ++++++++++++++++++ .../configuration/settings/settings_test.go | 6 +- internal/publicip/api/api.go | 50 +++--- internal/publicip/api/cloudflare.go | 4 + internal/publicip/api/ifconfigco.go | 4 + internal/publicip/api/interfaces.go | 11 ++ internal/publicip/api/ip2location.go | 4 + internal/publicip/api/ipinfo.go | 4 + internal/publicip/api/multi.go | 2 +- internal/publicip/api/resilient.go | 153 +++++++++++++++++ internal/publicip/interfaces.go | 13 -- internal/publicip/loop.go | 51 ++++-- internal/publicip/update.go | 11 ++ 22 files changed, 649 insertions(+), 90 deletions(-) create mode 100644 internal/configuration/settings/mocks_generate_test.go create mode 100644 internal/configuration/settings/mocks_reader_test.go create mode 100644 internal/configuration/settings/mocks_test.go create mode 100644 internal/configuration/settings/publicip_test.go create mode 100644 internal/publicip/api/resilient.go diff --git a/Dockerfile b/Dockerfile index 2ff0d7ba..ea19eea8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -205,7 +205,7 @@ ENV VPN_SERVICE_PROVIDER=pia \ # Public IP PUBLICIP_FILE="/tmp/gluetun/ip" \ PUBLICIP_ENABLED=on \ - PUBLICIP_API=ipinfo \ + PUBLICIP_API=ipinfo,ifconfigco,ip2location,cloudflare \ PUBLICIP_API_TOKEN= \ # Storage STORAGE_FILEPATH=/gluetun/servers.json \ diff --git a/cmd/gluetun/main.go b/cmd/gluetun/main.go index ad8f309e..d7a99366 100644 --- a/cmd/gluetun/main.go +++ b/cmd/gluetun/main.go @@ -34,7 +34,6 @@ import ( "github.com/qdm12/gluetun/internal/pprof" "github.com/qdm12/gluetun/internal/provider" "github.com/qdm12/gluetun/internal/publicip" - pubipapi "github.com/qdm12/gluetun/internal/publicip/api" "github.com/qdm12/gluetun/internal/routing" "github.com/qdm12/gluetun/internal/server" "github.com/qdm12/gluetun/internal/shadowsocks" @@ -407,14 +406,11 @@ func _main(ctx context.Context, buildInfo models.BuildInformation, go dnsLooper.RunRestartTicker(dnsTickerCtx, dnsTickerDone) controlGroupHandler.Add(dnsTickerHandler) - publicipAPI, _ := pubipapi.ParseProvider(allSettings.PublicIP.API) - ipFetcher, err := pubipapi.New(publicipAPI, httpClient, *allSettings.PublicIP.APIToken) + publicIPLooper, err := publicip.NewLoop(allSettings.PublicIP, puid, pgid, httpClient, + logger.New(log.SetComponent("ip getter"))) if err != nil { - return fmt.Errorf("creating public IP API client: %w", err) + return fmt.Errorf("creating public ip loop: %w", err) } - publicIPLooper := publicip.NewLoop(ipFetcher, - logger.New(log.SetComponent("ip getter")), - allSettings.PublicIP, puid, pgid) publicIPRunError, err := publicIPLooper.Start(ctx) if err != nil { return fmt.Errorf("starting public ip loop: %w", err) @@ -426,7 +422,7 @@ func _main(ctx context.Context, buildInfo models.BuildInformation, parallelResolver := resolver.NewParallelResolver(allSettings.Updater.DNSAddress) openvpnFileExtractor := extract.New() providers := provider.NewProviders(storage, time.Now, updaterLogger, - httpClient, unzipper, parallelResolver, ipFetcher, openvpnFileExtractor) + httpClient, unzipper, parallelResolver, publicIPLooper.Fetcher(), openvpnFileExtractor) vpnLogger := logger.New(log.SetComponent("vpn")) vpnLooper := vpn.NewLoop(allSettings.VPN, ipv6Supported, allSettings.Firewall.VPNInputPorts, diff --git a/internal/cli/update.go b/internal/cli/update.go index 5e4cdcfd..9e684490 100644 --- a/internal/cli/update.go +++ b/internal/cli/update.go @@ -80,10 +80,17 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e httpClient := &http.Client{Timeout: clientTimeout} unzipper := unzip.New(httpClient) parallelResolver := resolver.NewParallelResolver(options.DNSAddress) - ipFetcher, err := api.New(api.IPInfo, httpClient, ipToken) - if err != nil { - return fmt.Errorf("creating public IP API client: %w", err) + nameTokenPairs := []api.NameToken{ + {Name: string(api.IPInfo), Token: ipToken}, + {Name: string(api.IP2Location)}, + {Name: string(api.IfConfigCo)}, } + fetchers, err := api.New(nameTokenPairs, httpClient) + if err != nil { + return fmt.Errorf("creating public IP fetchers: %w", err) + } + ipFetcher := api.NewResilient(fetchers, logger) + openvpnFileExtractor := extract.New() providers := provider.NewProviders(storage, time.Now, logger, httpClient, diff --git a/internal/configuration/settings/helpers_test.go b/internal/configuration/settings/helpers_test.go index 346e7e9d..df93cd93 100644 --- a/internal/configuration/settings/helpers_test.go +++ b/internal/configuration/settings/helpers_test.go @@ -1,3 +1,30 @@ package settings -func boolPtr(b bool) *bool { return &b } +import gomock "github.com/golang/mock/gomock" + +type sourceKeyValue struct { + key string + value string +} + +func newMockSource(ctrl *gomock.Controller, keyValues []sourceKeyValue) *MockSource { + source := NewMockSource(ctrl) + var previousCall *gomock.Call + for _, keyValue := range keyValues { + transformedKey := keyValue.key + keyTransformCall := source.EXPECT().KeyTransform(keyValue.key).Return(transformedKey) + if previousCall != nil { + keyTransformCall.After(previousCall) + } + isSet := keyValue.value != "" + previousCall = source.EXPECT().Get(transformedKey). + Return(keyValue.value, isSet).After(keyTransformCall) + if isSet { + previousCall = source.EXPECT().KeyTransform(keyValue.key). + Return(transformedKey).After(previousCall) + previousCall = source.EXPECT().String(). + Return("mock source").After(previousCall) + } + } + return source +} diff --git a/internal/configuration/settings/mocks_generate_test.go b/internal/configuration/settings/mocks_generate_test.go new file mode 100644 index 00000000..80cc7482 --- /dev/null +++ b/internal/configuration/settings/mocks_generate_test.go @@ -0,0 +1,4 @@ +package settings + +//go:generate mockgen -destination=mocks_test.go -package=$GOPACKAGE . Warner +//go:generate mockgen -destination=mocks_reader_test.go -package=$GOPACKAGE github.com/qdm12/gosettings/reader Source diff --git a/internal/configuration/settings/mocks_reader_test.go b/internal/configuration/settings/mocks_reader_test.go new file mode 100644 index 00000000..b7a1e12f --- /dev/null +++ b/internal/configuration/settings/mocks_reader_test.go @@ -0,0 +1,77 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/qdm12/gosettings/reader (interfaces: Source) + +// Package settings is a generated GoMock package. +package settings + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockSource is a mock of Source interface. +type MockSource struct { + ctrl *gomock.Controller + recorder *MockSourceMockRecorder +} + +// MockSourceMockRecorder is the mock recorder for MockSource. +type MockSourceMockRecorder struct { + mock *MockSource +} + +// NewMockSource creates a new mock instance. +func NewMockSource(ctrl *gomock.Controller) *MockSource { + mock := &MockSource{ctrl: ctrl} + mock.recorder = &MockSourceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSource) EXPECT() *MockSourceMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockSource) Get(arg0 string) (string, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockSourceMockRecorder) Get(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockSource)(nil).Get), arg0) +} + +// KeyTransform mocks base method. +func (m *MockSource) KeyTransform(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeyTransform", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// KeyTransform indicates an expected call of KeyTransform. +func (mr *MockSourceMockRecorder) KeyTransform(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyTransform", reflect.TypeOf((*MockSource)(nil).KeyTransform), arg0) +} + +// String mocks base method. +func (m *MockSource) String() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String") + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockSourceMockRecorder) String() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockSource)(nil).String)) +} diff --git a/internal/configuration/settings/mocks_test.go b/internal/configuration/settings/mocks_test.go new file mode 100644 index 00000000..8a216c33 --- /dev/null +++ b/internal/configuration/settings/mocks_test.go @@ -0,0 +1,46 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/qdm12/gluetun/internal/configuration/settings (interfaces: Warner) + +// Package settings is a generated GoMock package. +package settings + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockWarner is a mock of Warner interface. +type MockWarner struct { + ctrl *gomock.Controller + recorder *MockWarnerMockRecorder +} + +// MockWarnerMockRecorder is the mock recorder for MockWarner. +type MockWarnerMockRecorder struct { + mock *MockWarner +} + +// NewMockWarner creates a new mock instance. +func NewMockWarner(ctrl *gomock.Controller) *MockWarner { + mock := &MockWarner{ctrl: ctrl} + mock.recorder = &MockWarnerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWarner) EXPECT() *MockWarnerMockRecorder { + return m.recorder +} + +// Warn mocks base method. +func (m *MockWarner) Warn(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Warn", arg0) +} + +// Warn indicates an expected call of Warn. +func (mr *MockWarnerMockRecorder) Warn(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Warn", reflect.TypeOf((*MockWarner)(nil).Warn), arg0) +} diff --git a/internal/configuration/settings/portforward_test.go b/internal/configuration/settings/portforward_test.go index 96ab68d4..bb79b446 100644 --- a/internal/configuration/settings/portforward_test.go +++ b/internal/configuration/settings/portforward_test.go @@ -10,7 +10,7 @@ func Test_PortForwarding_String(t *testing.T) { t.Parallel() settings := PortForwarding{ - Enabled: boolPtr(false), + Enabled: ptrTo(false), } s := settings.String() diff --git a/internal/configuration/settings/publicip.go b/internal/configuration/settings/publicip.go index 3ab654c5..7953bc57 100644 --- a/internal/configuration/settings/publicip.go +++ b/internal/configuration/settings/publicip.go @@ -20,15 +20,20 @@ type PublicIP struct { // to write to a file. It cannot be nil for the // internal state IPFilepath *string - // API is the API name to use to fetch public IP information. - // It can be cloudflare, ifconfigco, ip2location or ipinfo. - // It defaults to ipinfo. - API string - // APIToken is the token to use for the IP data service - // such as ipinfo.io. It can be the empty string to - // indicate not to use a token. It cannot be nil for the - // internal state. - APIToken *string + // APIs is the list of public ip APIs to use to fetch public IP information. + // If there is more than one API, the first one is used + // by default and the others are used as fallbacks in case of + // the service rate limiting us. It defaults to use all services, + // with the first one being ipinfo.io for historical reasons. + APIs []PublicIPAPI +} + +type PublicIPAPI struct { + // Name is the name of the public ip API service. + // It can be "cloudflare", "ifconfigco", "ip2location" or "ipinfo". + Name string + // Token is the token to use for the public ip API service. + Token string } // UpdateWith deep copies the receiving settings, overrides the copy with @@ -53,9 +58,11 @@ func (p PublicIP) validate() (err error) { } } - _, err = api.ParseProvider(p.API) - if err != nil { - return fmt.Errorf("API name: %w", err) + for _, publicIPAPI := range p.APIs { + _, err = api.ParseProvider(publicIPAPI.Name) + if err != nil { + return fmt.Errorf("API name: %w", err) + } } return nil @@ -65,23 +72,25 @@ func (p *PublicIP) copy() (copied PublicIP) { return PublicIP{ Enabled: gosettings.CopyPointer(p.Enabled), IPFilepath: gosettings.CopyPointer(p.IPFilepath), - API: p.API, - APIToken: gosettings.CopyPointer(p.APIToken), + APIs: gosettings.CopySlice(p.APIs), } } func (p *PublicIP) overrideWith(other PublicIP) { p.Enabled = gosettings.OverrideWithPointer(p.Enabled, other.Enabled) p.IPFilepath = gosettings.OverrideWithPointer(p.IPFilepath, other.IPFilepath) - p.API = gosettings.OverrideWithComparable(p.API, other.API) - p.APIToken = gosettings.OverrideWithPointer(p.APIToken, other.APIToken) + p.APIs = gosettings.OverrideWithSlice(p.APIs, other.APIs) } func (p *PublicIP) setDefaults() { p.Enabled = gosettings.DefaultPointer(p.Enabled, true) p.IPFilepath = gosettings.DefaultPointer(p.IPFilepath, "/tmp/gluetun/ip") - p.API = gosettings.DefaultComparable(p.API, "ipinfo") - p.APIToken = gosettings.DefaultPointer(p.APIToken, "") + p.APIs = gosettings.DefaultSlice(p.APIs, []PublicIPAPI{ + {Name: string(api.IPInfo)}, + {Name: string(api.Cloudflare)}, + {Name: string(api.IfConfigCo)}, + {Name: string(api.IP2Location)}, + }) } func (p PublicIP) String() string { @@ -99,10 +108,20 @@ func (p PublicIP) toLinesNode() (node *gotree.Node) { node.Appendf("IP file path: %s", *p.IPFilepath) } - node.Appendf("Public IP data API: %s", p.API) - - if *p.APIToken != "" { - node.Appendf("API token: %s", gosettings.ObfuscateKey(*p.APIToken)) + baseAPIString := "Public IP data base API: " + p.APIs[0].Name + if p.APIs[0].Token != "" { + baseAPIString += " (token " + gosettings.ObfuscateKey(p.APIs[0].Token) + ")" + } + node.Append(baseAPIString) + if len(p.APIs) > 1 { + backupAPIsNode := node.Append("Public IP data backup APIs:") + for i := 1; i < len(p.APIs); i++ { + message := p.APIs[i].Name + if p.APIs[i].Token != "" { + message += " (token " + gosettings.ObfuscateKey(p.APIs[i].Token) + ")" + } + backupAPIsNode.Append(message) + } } return node @@ -116,8 +135,21 @@ func (p *PublicIP) read(r *reader.Reader, warner Warner) (err error) { p.IPFilepath = r.Get("PUBLICIP_FILE", reader.ForceLowercase(false), reader.RetroKeys("IP_STATUS_FILE")) - p.API = r.String("PUBLICIP_API") - p.APIToken = r.Get("PUBLICIP_API_TOKEN") + + apiNames := r.CSV("PUBLICIP_API") + if len(apiNames) > 0 { + apiTokens := r.CSV("PUBLICIP_API_TOKEN") + p.APIs = make([]PublicIPAPI, len(apiNames)) + for i := range apiNames { + p.APIs[i].Name = apiNames[i] + var token string + if i < len(apiTokens) { // only set token if it exists + token = apiTokens[i] + } + p.APIs[i].Token = token + } + } + return nil } diff --git a/internal/configuration/settings/publicip_test.go b/internal/configuration/settings/publicip_test.go new file mode 100644 index 00000000..1195af5d --- /dev/null +++ b/internal/configuration/settings/publicip_test.go @@ -0,0 +1,161 @@ +package settings + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/qdm12/gosettings/reader" + "github.com/stretchr/testify/assert" +) + +func Test_PublicIP_read(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + makeReader func(ctrl *gomock.Controller) *reader.Reader + makeWarner func(ctrl *gomock.Controller) Warner + settings PublicIP + errWrapped error + errMessage string + }{ + "nothing_read": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + }, + "single_api_no_token": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API", value: "ipinfo"}, + {key: "PUBLICIP_API_TOKEN"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + settings: PublicIP{ + APIs: []PublicIPAPI{ + {Name: "ipinfo"}, + }, + }, + }, + "single_api_with_token": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API", value: "ipinfo"}, + {key: "PUBLICIP_API_TOKEN", value: "xyz"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + settings: PublicIP{ + APIs: []PublicIPAPI{ + {Name: "ipinfo", Token: "xyz"}, + }, + }, + }, + "multiple_apis_no_token": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API", value: "ipinfo,ip2location"}, + {key: "PUBLICIP_API_TOKEN"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + settings: PublicIP{ + APIs: []PublicIPAPI{ + {Name: "ipinfo"}, + {Name: "ip2location"}, + }, + }, + }, + "multiple_apis_with_token": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API", value: "ipinfo,ip2location"}, + {key: "PUBLICIP_API_TOKEN", value: "xyz,abc"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + settings: PublicIP{ + APIs: []PublicIPAPI{ + {Name: "ipinfo", Token: "xyz"}, + {Name: "ip2location", Token: "abc"}, + }, + }, + }, + "multiple_apis_with_and_without_token": { + makeReader: func(ctrl *gomock.Controller) *reader.Reader { + source := newMockSource(ctrl, []sourceKeyValue{ + {key: "PUBLICIP_PERIOD"}, + {key: "PUBLICIP_ENABLED"}, + {key: "IP_STATUS_FILE"}, + {key: "PUBLICIP_FILE"}, + {key: "PUBLICIP_API", value: "ipinfo,ip2location"}, + {key: "PUBLICIP_API_TOKEN", value: "xyz"}, + }) + return reader.New(reader.Settings{ + Sources: []reader.Source{source}, + }) + }, + settings: PublicIP{ + APIs: []PublicIPAPI{ + {Name: "ipinfo", Token: "xyz"}, + {Name: "ip2location"}, + }, + }, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + + reader := testCase.makeReader(ctrl) + var warner Warner + if testCase.makeWarner != nil { + warner = testCase.makeWarner(ctrl) + } + + var settings PublicIP + err := settings.read(reader, warner) + + assert.Equal(t, testCase.settings, settings) + assert.ErrorIs(t, err, testCase.errWrapped) + if testCase.errWrapped != nil { + assert.EqualError(t, err, testCase.errMessage) + } + }) + } +} diff --git a/internal/configuration/settings/settings_test.go b/internal/configuration/settings/settings_test.go index 75ae9292..7aa30a15 100644 --- a/internal/configuration/settings/settings_test.go +++ b/internal/configuration/settings/settings_test.go @@ -79,7 +79,11 @@ func Test_Settings_String(t *testing.T) { | └── Process GID: 1000 ├── Public IP settings: | ├── IP file path: /tmp/gluetun/ip -| └── Public IP data API: ipinfo +| ├── Public IP data base API: ipinfo +| └── Public IP data backup APIs: +| ├── cloudflare +| ├── ifconfigco +| └── ip2location └── Version settings: └── Enabled: yes`, }, diff --git a/internal/publicip/api/api.go b/internal/publicip/api/api.go index 623b1c6c..e7774a6c 100644 --- a/internal/publicip/api/api.go +++ b/internal/publicip/api/api.go @@ -1,23 +1,12 @@ package api import ( - "context" "errors" "fmt" "net/http" - "net/netip" "strings" - - "github.com/qdm12/gluetun/internal/models" ) -type API interface { - String() string - CanFetchAnyIP() bool - FetchInfo(ctx context.Context, ip netip.Addr) ( - result models.PublicIP, err error) -} - type Provider string const ( @@ -27,21 +16,34 @@ const ( IP2Location Provider = "ip2location" ) -func New(provider Provider, client *http.Client, token string) ( //nolint:ireturn - a API, err error, +type NameToken struct { + Name string + Token string +} + +func New(nameTokenPairs []NameToken, client *http.Client) ( + fetchers []Fetcher, err error, ) { - switch provider { - case Cloudflare: - return newCloudflare(client), nil - case IfConfigCo: - return newIfConfigCo(client), nil - case IPInfo: - return newIPInfo(client, token), nil - case IP2Location: - return newIP2Location(client, token), nil - default: - panic("provider not valid: " + provider) + fetchers = make([]Fetcher, len(nameTokenPairs)) + for i, nameTokenPair := range nameTokenPairs { + provider, err := ParseProvider(nameTokenPair.Name) + if err != nil { + return nil, fmt.Errorf("parsing API name: %w", err) + } + switch provider { + case Cloudflare: + fetchers[i] = newCloudflare(client) + case IfConfigCo: + fetchers[i] = newIfConfigCo(client) + case IPInfo: + fetchers[i] = newIPInfo(client, nameTokenPair.Token) + case IP2Location: + fetchers[i] = newIP2Location(client, nameTokenPair.Token) + default: + panic("provider not valid: " + provider) + } } + return fetchers, nil } var ErrProviderNotValid = errors.New("API name is not valid") diff --git a/internal/publicip/api/cloudflare.go b/internal/publicip/api/cloudflare.go index 56a5f03a..6421b129 100644 --- a/internal/publicip/api/cloudflare.go +++ b/internal/publicip/api/cloudflare.go @@ -30,6 +30,10 @@ func (c *cloudflare) CanFetchAnyIP() bool { return false } +func (c *cloudflare) Token() (token string) { + return "" +} + // FetchInfo obtains information on the public IP address of the machine, // and returns an error if the `ip` argument is set since the Cloudflare API // can only be used to provide details about the current machine public IP. diff --git a/internal/publicip/api/ifconfigco.go b/internal/publicip/api/ifconfigco.go index 8da07996..c0d8fdcb 100644 --- a/internal/publicip/api/ifconfigco.go +++ b/internal/publicip/api/ifconfigco.go @@ -28,6 +28,10 @@ func (i *ifConfigCo) CanFetchAnyIP() bool { return true } +func (i *ifConfigCo) Token() string { + return "" +} + // FetchInfo obtains information on the ip address provided // using the ifconfig.co/json API. If the ip is the zero value, // the public IP address of the machine is used as the IP. diff --git a/internal/publicip/api/interfaces.go b/internal/publicip/api/interfaces.go index 76051d14..056a026c 100644 --- a/internal/publicip/api/interfaces.go +++ b/internal/publicip/api/interfaces.go @@ -8,6 +8,17 @@ import ( ) type Fetcher interface { + String() string + CanFetchAnyIP() bool + Token() (token string) + InfoFetcher +} + +type InfoFetcher interface { FetchInfo(ctx context.Context, ip netip.Addr) ( result models.PublicIP, err error) } + +type Warner interface { + Warn(message string) +} diff --git a/internal/publicip/api/ip2location.go b/internal/publicip/api/ip2location.go index 7a3174e7..35ac5f51 100644 --- a/internal/publicip/api/ip2location.go +++ b/internal/publicip/api/ip2location.go @@ -31,6 +31,10 @@ func (i *ip2Location) CanFetchAnyIP() bool { return true } +func (i *ip2Location) Token() string { + return i.token +} + // FetchInfo obtains information on the ip address provided // using the api.ip2location.io API. If the ip is the zero value, // the public IP address of the machine is used as the IP. diff --git a/internal/publicip/api/ipinfo.go b/internal/publicip/api/ipinfo.go index 4ff78785..300ec9c2 100644 --- a/internal/publicip/api/ipinfo.go +++ b/internal/publicip/api/ipinfo.go @@ -32,6 +32,10 @@ func (i *ipInfo) CanFetchAnyIP() bool { return true } +func (i *ipInfo) Token() string { + return i.token +} + // FetchInfo obtains information on the ip address provided // using the ipinfo.io API. If the ip is the zero value, the public IP address // of the machine is used as the IP. diff --git a/internal/publicip/api/multi.go b/internal/publicip/api/multi.go index ab46f9ee..a810a0e3 100644 --- a/internal/publicip/api/multi.go +++ b/internal/publicip/api/multi.go @@ -13,7 +13,7 @@ import ( // If an error is encountered, all the operations are canceled and // an error is returned, so the results returned should be considered // incomplete in this case. -func FetchMultiInfo(ctx context.Context, fetcher Fetcher, ips []netip.Addr) ( +func FetchMultiInfo(ctx context.Context, fetcher InfoFetcher, ips []netip.Addr) ( results []models.PublicIP, err error, ) { ctx, cancel := context.WithCancel(ctx) diff --git a/internal/publicip/api/resilient.go b/internal/publicip/api/resilient.go new file mode 100644 index 00000000..0aefd677 --- /dev/null +++ b/internal/publicip/api/resilient.go @@ -0,0 +1,153 @@ +package api + +import ( + "context" + "errors" + "fmt" + "net/netip" + "strings" + "sync" + "time" + + "github.com/qdm12/gluetun/internal/models" +) + +type ResilientFetcher struct { + fetchers []Fetcher + logger Warner + fetcherToBanTime map[Fetcher]time.Time + mutex sync.RWMutex + timeNow func() time.Time +} + +// NewResilient creates a 'resilient' fetcher given multiple fetchers. +// For example, it can handle bans and move on to another fetcher if one fails. +func NewResilient(fetchers []Fetcher, logger Warner) *ResilientFetcher { + return &ResilientFetcher{ + fetchers: fetchers, + logger: logger, + fetcherToBanTime: make(map[Fetcher]time.Time, len(fetchers)), + timeNow: time.Now, + } +} + +func (r *ResilientFetcher) isBanned(fetcher Fetcher) (banned bool) { + banTime, banned := r.fetcherToBanTime[fetcher] + if !banned { + return false + } + const banDuration = 30 * 24 * time.Hour + banExpiryTime := banTime.Add(banDuration) + now := r.timeNow() + if now.After(banExpiryTime) { + delete(r.fetcherToBanTime, fetcher) + return false + } + return true +} + +func (r *ResilientFetcher) String() string { + r.mutex.RLock() + defer r.mutex.RUnlock() + for _, fetcher := range r.fetchers { + if r.isBanned(fetcher) { + continue + } + return fetcher.String() + } + return "" +} + +func (r *ResilientFetcher) Token() string { + r.mutex.RLock() + defer r.mutex.RUnlock() + for _, fetcher := range r.fetchers { + if r.isBanned(fetcher) { + continue + } + return fetcher.Token() + } + return "" +} + +// CanFetchAnyIP returns true if any of the fetchers +// can fetch any IP address and is not banned. +func (r *ResilientFetcher) CanFetchAnyIP() bool { + r.mutex.RLock() + defer r.mutex.RUnlock() + + for _, fetcher := range r.fetchers { + if !fetcher.CanFetchAnyIP() || r.isBanned(fetcher) { + continue + } + return true + } + return false +} + +var ErrFetchersAllRateLimited = errors.New("all fetchers are rate limited") + +// FetchInfo obtains information on the ip address provided. +// If the ip is the zero value, the public IP address of the machine +// is used as the IP. +// If a fetcher gets banned, the next one is tried – until all have been exhausted. +// Fetchers still within their banned period are skipped. +// If an error unrelated to being banned is encountered, it is returned and more +// fetchers are tried. +func (r *ResilientFetcher) FetchInfo(ctx context.Context, ip netip.Addr) ( + result models.PublicIP, err error, +) { + r.mutex.RLock() + defer r.mutex.RUnlock() + + for _, fetcher := range r.fetchers { + if r.isBanned(fetcher) || + (ip.IsValid() && !fetcher.CanFetchAnyIP()) { + continue + } + + result, err = fetcher.FetchInfo(ctx, ip) + if err == nil || !errors.Is(err, ErrTooManyRequests) { + return result, err + } + + // Fetcher is banned + r.fetcherToBanTime[fetcher] = r.timeNow() + r.logger.Warn(fetcher.String() + ": " + err.Error()) + } + + fetcherNames := make([]string, len(r.fetchers)) + for i, fetcher := range r.fetchers { + fetcherNames[i] = fetcher.String() + } + + return result, fmt.Errorf("%w (%s)", + ErrFetchersAllRateLimited, + strings.Join(fetcherNames, ", ")) +} + +func (r *ResilientFetcher) UpdateFetchers(fetchers []Fetcher) { + newFetcherNameToFetcher := make(map[string]Fetcher, len(fetchers)) + for _, fetcher := range fetchers { + newFetcherNameToFetcher[fetcher.String()] = fetcher + } + + r.mutex.Lock() + defer r.mutex.Unlock() + + newFetcherToBanTime := make(map[Fetcher]time.Time, len(r.fetcherToBanTime)) + for bannedFetcher, banTime := range r.fetcherToBanTime { + if !r.isBanned(bannedFetcher) { + // fetcher is no longer in its ban period. + continue + } + bannedName := bannedFetcher.String() + newFetcher, isNewFetcher := newFetcherNameToFetcher[bannedName] + if isNewFetcher && newFetcher.Token() == bannedFetcher.Token() { + newFetcherToBanTime[newFetcher] = banTime + } + } + + r.fetchers = fetchers + r.fetcherToBanTime = newFetcherToBanTime +} diff --git a/internal/publicip/interfaces.go b/internal/publicip/interfaces.go index f0c0d4a9..017d641b 100644 --- a/internal/publicip/interfaces.go +++ b/internal/publicip/interfaces.go @@ -1,18 +1,5 @@ package publicip -import ( - "context" - "net/netip" - - "github.com/qdm12/gluetun/internal/models" -) - -type Fetcher interface { - String() string - FetchInfo(ctx context.Context, ip netip.Addr) ( - result models.PublicIP, err error) -} - type Logger interface { Info(s string) Warn(s string) diff --git a/internal/publicip/loop.go b/internal/publicip/loop.go index 0e383053..099224e9 100644 --- a/internal/publicip/loop.go +++ b/internal/publicip/loop.go @@ -3,12 +3,14 @@ package publicip import ( "context" "fmt" + "net/http" "net/netip" "sync" "time" "github.com/qdm12/gluetun/internal/configuration/settings" "github.com/qdm12/gluetun/internal/models" + "github.com/qdm12/gluetun/internal/publicip/api" ) type Loop struct { @@ -17,9 +19,10 @@ type Loop struct { settingsMutex sync.RWMutex ipData models.PublicIP ipDataMutex sync.RWMutex + fetcher *api.ResilientFetcher // Fixed injected objects - fetcher Fetcher - logger Logger + httpClient *http.Client + logger Logger // Fixed parameters puid int pgid int @@ -37,17 +40,34 @@ type Loop struct { timeNow func() time.Time } -func NewLoop(fetcher Fetcher, logger Logger, - settings settings.PublicIP, puid, pgid int, -) *Loop { - return &Loop{ - settings: settings, - fetcher: fetcher, - logger: logger, - puid: puid, - pgid: pgid, - timeNow: time.Now, +func NewLoop(settings settings.PublicIP, puid, pgid int, + httpClient *http.Client, logger Logger, +) (loop *Loop, err error) { + fetchers, err := api.New(makeNameTokenPairs(settings.APIs), httpClient) + if err != nil { + return nil, fmt.Errorf("creating fetchers: %w", err) } + + return &Loop{ + settings: settings, + httpClient: httpClient, + fetcher: api.NewResilient(fetchers, logger), + logger: logger, + puid: puid, + pgid: pgid, + timeNow: time.Now, + }, nil +} + +func makeNameTokenPairs(apis []settings.PublicIPAPI) (nameTokenPairs []api.NameToken) { + nameTokenPairs = make([]api.NameToken, len(apis)) + for i := range apis { + nameTokenPairs[i] = api.NameToken{ + Name: apis[i].Name, + Token: apis[i].Token, + } + } + return nameTokenPairs } func (l *Loop) String() string { @@ -102,7 +122,8 @@ func (l *Loop) run(runCtx context.Context, runDone chan<- struct{}, } message := "Public IP address is " + result.IP.String() - message += " (" + result.Country + ", " + result.Region + ", " + result.City + ")" + message += " (" + result.Country + ", " + result.Region + ", " + result.City + + " - source: " + l.fetcher.String() + ")" l.logger.Info(message) l.ipDataMutex.Lock() @@ -159,3 +180,7 @@ func (l *Loop) Stop() (err error) { <-l.runDone return l.ClearData() } + +func (l *Loop) Fetcher() (fetcher *api.ResilientFetcher) { + return l.fetcher +} diff --git a/internal/publicip/update.go b/internal/publicip/update.go index 4ca690a9..71dbdb13 100644 --- a/internal/publicip/update.go +++ b/internal/publicip/update.go @@ -3,8 +3,10 @@ package publicip import ( "fmt" "os" + "reflect" "github.com/qdm12/gluetun/internal/configuration/settings" + "github.com/qdm12/gluetun/internal/publicip/api" ) func (l *Loop) update(partialUpdate settings.PublicIP) (err error) { @@ -37,6 +39,15 @@ func (l *Loop) update(partialUpdate settings.PublicIP) (err error) { } } + if !reflect.DeepEqual(l.settings.APIs, updatedSettings.APIs) { + newFetchers, err := api.New(makeNameTokenPairs(updatedSettings.APIs), l.httpClient) + if err != nil { + return fmt.Errorf("creating fetchers: %w", err) + } + + l.fetcher.UpdateFetchers(newFetchers) + } + l.settings = updatedSettings return nil