chore(settings): refactor settings processing (#756)

- Better settings tree structure logged using `qdm12/gotree`
- Read settings from environment variables, then files, then secret files
- Settings methods to default them, merge them and override them
- `DNS_PLAINTEXT_ADDRESS` default changed to `127.0.0.1` to use DoT. Warning added if set to something else.
- `HTTPPROXY_LISTENING_ADDRESS` instead of `HTTPPROXY_PORT` (with retro-compatibility)

internal/updater/loop.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/qdm12/gluetun/internal/configuration"
+	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/storage"
@@ -18,8 +18,8 @@ type Looper interface {
 	GetStatus() (status models.LoopStatus)
 	SetStatus(ctx context.Context, status models.LoopStatus) (
 		outcome string, err error)
-	GetSettings() (settings configuration.Updater)
-	SetSettings(settings configuration.Updater) (outcome string)
+	GetSettings() (settings settings.Updater)
+	SetSettings(settings settings.Updater) (outcome string)
 }
 
 type looper struct {
@@ -44,7 +44,7 @@ type looper struct {
 
 const defaultBackoffTime = 5 * time.Second
 
-func NewLooper(settings configuration.Updater, currentServers models.AllServers,
+func NewLooper(settings settings.Updater, currentServers models.AllServers,
 	flusher storage.Flusher, setAllServers func(allServers models.AllServers),
 	client *http.Client, logger Logger) Looper {
 	return &looper{
@@ -164,7 +164,7 @@ func (l *looper) RunRestartTicker(ctx context.Context, done chan<- struct{}) {
 	timer := time.NewTimer(time.Hour)
 	timer.Stop()
 	timerIsStopped := true
-	if period := l.GetSettings().Period; period > 0 {
+	if period := *l.GetSettings().Period; period > 0 {
 		timerIsStopped = false
 		timer.Reset(period)
 	}
@@ -179,13 +179,13 @@ func (l *looper) RunRestartTicker(ctx context.Context, done chan<- struct{}) {
 		case <-timer.C:
 			lastTick = l.timeNow()
 			l.start <- struct{}{}
-			timer.Reset(l.GetSettings().Period)
+			timer.Reset(*l.GetSettings().Period)
 		case <-l.updateTicker:
 			if !timerIsStopped && !timer.Stop() {
 				<-timer.C
 			}
 			timerIsStopped = true
-			period := l.GetSettings().Period
+			period := *l.GetSettings().Period
 			if period == 0 {
 				continue
 			}

internal/updater/providers.go

@@ -48,7 +48,7 @@ func (u *updater) updateExpressvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Expressvpn.Servers))
 	servers, warnings, err := expressvpn.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("ExpressVPN: " + warning)
 		}
@@ -70,7 +70,7 @@ func (u *updater) updateFastestvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Fastestvpn.Servers))
 	servers, warnings, err := fastestvpn.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("FastestVPN: " + warning)
 		}
@@ -92,7 +92,7 @@ func (u *updater) updateHideMyAss(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.HideMyAss.Servers))
 	servers, warnings, err := hidemyass.GetServers(
 		ctx, u.client, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("HideMyAss: " + warning)
 		}
@@ -114,7 +114,7 @@ func (u *updater) updateIpvanish(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Ipvanish.Servers))
 	servers, warnings, err := ipvanish.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("Ipvanish: " + warning)
 		}
@@ -136,7 +136,7 @@ func (u *updater) updateIvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Ivpn.Servers))
 	servers, warnings, err := ivpn.GetServers(
 		ctx, u.client, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("Ivpn: " + warning)
 		}
@@ -173,7 +173,7 @@ func (u *updater) updateMullvad(ctx context.Context) (err error) {
 func (u *updater) updateNordvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Nordvpn.Servers))
 	servers, warnings, err := nordvpn.GetServers(ctx, u.client, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("NordVPN: " + warning)
 		}
@@ -194,7 +194,7 @@ func (u *updater) updateNordvpn(ctx context.Context) (err error) {
 func (u *updater) updatePerfectprivacy(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Perfectprivacy.Servers))
 	servers, warnings, err := perfectprivacy.GetServers(ctx, u.unzipper, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn(constants.Perfectprivacy + ": " + warning)
 		}
@@ -232,7 +232,7 @@ func (u *updater) updatePrivado(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Privado.Servers))
 	servers, warnings, err := privado.GetServers(
 		ctx, u.unzipper, u.client, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("Privado: " + warning)
 		}
@@ -254,7 +254,7 @@ func (u *updater) updatePrivatevpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Privatevpn.Servers))
 	servers, warnings, err := privatevpn.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("PrivateVPN: " + warning)
 		}
@@ -275,7 +275,7 @@ func (u *updater) updatePrivatevpn(ctx context.Context) (err error) {
 func (u *updater) updateProtonvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Privatevpn.Servers))
 	servers, warnings, err := protonvpn.GetServers(ctx, u.client, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("ProtonVPN: " + warning)
 		}
@@ -297,7 +297,7 @@ func (u *updater) updatePurevpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Purevpn.Servers))
 	servers, warnings, err := purevpn.GetServers(
 		ctx, u.client, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("PureVPN: " + warning)
 		}
@@ -319,7 +319,7 @@ func (u *updater) updateSurfshark(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Surfshark.Servers))
 	servers, warnings, err := surfshark.GetServers(
 		ctx, u.unzipper, u.client, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("Surfshark: " + warning)
 		}
@@ -341,7 +341,7 @@ func (u *updater) updateTorguard(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Torguard.Servers))
 	servers, warnings, err := torguard.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("Torguard: " + warning)
 		}
@@ -363,7 +363,7 @@ func (u *updater) updateVPNUnlimited(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.VPNUnlimited.Servers))
 	servers, warnings, err := vpnunlimited.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn(constants.VPNUnlimited + ": " + warning)
 		}
@@ -385,7 +385,7 @@ func (u *updater) updateVyprvpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Vyprvpn.Servers))
 	servers, warnings, err := vyprvpn.GetServers(
 		ctx, u.unzipper, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("VyprVPN: " + warning)
 		}
@@ -406,7 +406,7 @@ func (u *updater) updateVyprvpn(ctx context.Context) (err error) {
 func (u *updater) updateWevpn(ctx context.Context) (err error) {
 	minServers := getMinServers(len(u.servers.Wevpn.Servers))
 	servers, warnings, err := wevpn.GetServers(ctx, u.presolver, minServers)
-	if u.options.CLI {
+	if *u.options.CLI {
 		for _, warning := range warnings {
 			u.logger.Warn("WeVPN: " + warning)
 		}

internal/updater/state.go

@@ -7,14 +7,14 @@ import (
 	"reflect"
 	"sync"
 
-	"github.com/qdm12/gluetun/internal/configuration"
+	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/models"
 )
 
 type state struct {
 	status   models.LoopStatus
-	settings configuration.Updater
+	settings settings.Updater
 	statusMu sync.RWMutex
 	periodMu sync.RWMutex
 }
@@ -84,13 +84,13 @@ func (l *looper) SetStatus(ctx context.Context, status models.LoopStatus) (outco
 	}
 }
 
-func (l *looper) GetSettings() (settings configuration.Updater) {
+func (l *looper) GetSettings() (settings settings.Updater) {
 	l.state.periodMu.RLock()
 	defer l.state.periodMu.RUnlock()
 	return l.state.settings
 }
 
-func (l *looper) SetSettings(settings configuration.Updater) (outcome string) {
+func (l *looper) SetSettings(settings settings.Updater) (outcome string) {
 	l.state.periodMu.Lock()
 	defer l.state.periodMu.Unlock()
 	settingsUnchanged := reflect.DeepEqual(settings, l.state.settings)

internal/updater/updater.go

@@ -4,9 +4,10 @@ package updater
 import (
 	"context"
 	"net/http"
+	"strings"
 	"time"
 
-	"github.com/qdm12/gluetun/internal/configuration"
+	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/updater/resolver"
@@ -19,7 +20,7 @@ type Updater interface {
 
 type updater struct {
 	// configuration
-	options configuration.Updater
+	options settings.Updater
 
 	// state
 	servers models.AllServers
@@ -32,16 +33,13 @@ type updater struct {
 	unzipper  unzip.Unzipper
 }
 
-func New(settings configuration.Updater, httpClient *http.Client,
+func New(settings settings.Updater, httpClient *http.Client,
 	currentServers models.AllServers, logger Logger) Updater {
-	if settings.DNSAddress == "" {
-		settings.DNSAddress = "1.1.1.1"
-	}
 	unzipper := unzip.New(httpClient)
 	return &updater{
 		logger:    logger,
 		timeNow:   time.Now,
-		presolver: resolver.NewParallelResolver(settings.DNSAddress),
+		presolver: resolver.NewParallelResolver(settings.DNSAddress.String()),
 		client:    httpClient,
 		unzipper:  unzipper,
 		options:   settings,
@@ -49,203 +47,17 @@ func New(settings configuration.Updater, httpClient *http.Client,
 	}
 }
 
-//nolint:gocognit,gocyclo
+type updateFunc func(ctx context.Context) (err error)
+
 func (u *updater) UpdateServers(ctx context.Context) (allServers models.AllServers, err error) {
-	if u.options.Cyberghost {
-		u.logger.Info("updating Cyberghost servers...")
-		if err := u.updateCyberghost(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
+	for _, provider := range u.options.Providers {
+		u.logger.Info("updating " + strings.Title(provider) + " servers...")
+		updateProvider := u.getUpdateFunction(provider)
 
-	if u.options.Expressvpn {
-		u.logger.Info("updating Expressvpn servers...")
-		if err := u.updateExpressvpn(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Fastestvpn {
-		u.logger.Info("updating Fastestvpn servers...")
-		if err := u.updateFastestvpn(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.HideMyAss {
-		u.logger.Info("updating HideMyAss servers...")
-		if err := u.updateHideMyAss(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Ipvanish {
-		u.logger.Info("updating Ipvanish servers...")
-		if err := u.updateIpvanish(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Ivpn {
-		u.logger.Info("updating Ivpn servers...")
-		if err := u.updateIvpn(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Mullvad {
-		u.logger.Info("updating Mullvad servers...")
-		if err := u.updateMullvad(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Nordvpn {
 		// TODO support servers offering only TCP or only UDP
-		u.logger.Info("updating NordVPN servers...")
-		if err := u.updateNordvpn(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if err := ctx.Err(); err != nil {
-			return allServers, err
-		}
-	}
-
-	if u.options.Perfectprivacy {
-		u.logger.Info("updating " + constants.Perfectprivacy + " servers...")
-		if err := u.updatePerfectprivacy(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Privado {
-		u.logger.Info("updating Privado servers...")
-		if err := u.updatePrivado(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if ctx.Err() != nil {
-			return allServers, ctx.Err()
-		}
-	}
-
-	if u.options.PIA {
-		u.logger.Info("updating Private Internet Access servers...")
-		if err := u.updatePIA(ctx); err != nil {
-			u.logger.Error(err.Error())
-		}
-		if ctx.Err() != nil {
-			return allServers, ctx.Err()
-		}
-	}
-
-	if u.options.Privatevpn {
-		u.logger.Info("updating Privatevpn servers...")
-		if err := u.updatePrivatevpn(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Protonvpn {
-		u.logger.Info("updating Protonvpn servers...")
-		if err := u.updateProtonvpn(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Purevpn {
-		u.logger.Info("updating PureVPN servers...")
-		// TODO support servers offering only TCP or only UDP
-		if err := u.updatePurevpn(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Surfshark {
-		u.logger.Info("updating Surfshark servers...")
-		if err := u.updateSurfshark(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Torguard {
-		u.logger.Info("updating Torguard servers...")
-		if err := u.updateTorguard(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.VPNUnlimited {
-		u.logger.Info("updating " + constants.VPNUnlimited + " servers...")
-		if err := u.updateVPNUnlimited(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Vyprvpn {
-		u.logger.Info("updating Vyprvpn servers...")
-		if err := u.updateVyprvpn(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Wevpn {
-		u.logger.Info("updating WeVPN servers...")
-		if err := u.updateWevpn(ctx); err != nil {
-			if ctxErr := ctx.Err(); ctxErr != nil {
-				return allServers, ctxErr
-			}
-			u.logger.Error(err.Error())
-		}
-	}
-
-	if u.options.Windscribe {
-		u.logger.Info("updating Windscribe servers...")
-		if err := u.updateWindscribe(ctx); err != nil {
+		// for NordVPN and PureVPN
+		err = updateProvider(ctx)
+		if err != nil {
 			if ctxErr := ctx.Err(); ctxErr != nil {
 				return allServers, ctxErr
 			}
@@ -255,3 +67,52 @@ func (u *updater) UpdateServers(ctx context.Context) (allServers models.AllServe
 
 	return u.servers, nil
 }
+
+func (u *updater) getUpdateFunction(provider string) (updateFunction updateFunc) {
+	switch provider {
+	case constants.Custom:
+		panic("cannot update custom provider")
+	case constants.Cyberghost:
+		return func(ctx context.Context) (err error) { return u.updateCyberghost(ctx) }
+	case constants.Expressvpn:
+		return func(ctx context.Context) (err error) { return u.updateExpressvpn(ctx) }
+	case constants.Fastestvpn:
+		return func(ctx context.Context) (err error) { return u.updateFastestvpn(ctx) }
+	case constants.HideMyAss:
+		return func(ctx context.Context) (err error) { return u.updateHideMyAss(ctx) }
+	case constants.Ipvanish:
+		return func(ctx context.Context) (err error) { return u.updateIpvanish(ctx) }
+	case constants.Ivpn:
+		return func(ctx context.Context) (err error) { return u.updateIvpn(ctx) }
+	case constants.Mullvad:
+		return func(ctx context.Context) (err error) { return u.updateMullvad(ctx) }
+	case constants.Nordvpn:
+		return func(ctx context.Context) (err error) { return u.updateNordvpn(ctx) }
+	case constants.Perfectprivacy:
+		return func(ctx context.Context) (err error) { return u.updatePerfectprivacy(ctx) }
+	case constants.Privado:
+		return func(ctx context.Context) (err error) { return u.updatePrivado(ctx) }
+	case constants.PrivateInternetAccess:
+		return func(ctx context.Context) (err error) { return u.updatePIA(ctx) }
+	case constants.Privatevpn:
+		return func(ctx context.Context) (err error) { return u.updatePrivatevpn(ctx) }
+	case constants.Protonvpn:
+		return func(ctx context.Context) (err error) { return u.updateProtonvpn(ctx) }
+	case constants.Purevpn:
+		return func(ctx context.Context) (err error) { return u.updatePurevpn(ctx) }
+	case constants.Surfshark:
+		return func(ctx context.Context) (err error) { return u.updateSurfshark(ctx) }
+	case constants.Torguard:
+		return func(ctx context.Context) (err error) { return u.updateTorguard(ctx) }
+	case constants.VPNUnlimited:
+		return func(ctx context.Context) (err error) { return u.updateVPNUnlimited(ctx) }
+	case constants.Vyprvpn:
+		return func(ctx context.Context) (err error) { return u.updateVyprvpn(ctx) }
+	case constants.Wevpn:
+		return func(ctx context.Context) (err error) { return u.updateWevpn(ctx) }
+	case constants.Windscribe:
+		return func(ctx context.Context) (err error) { return u.updateWindscribe(ctx) }
+	default:
+		panic("provider " + provider + " is unknown")
+	}
+}