feat(dns): replace unbound with qdm12/dns@v2.0.0-beta-rc6 (#1742)

- Faster start up
- Clearer error messages
- Allow for more Gluetun-specific customization
- DNSSEC validation is dropped for now (it's sort of unneeded)
- Fix #137

internal/dns/interfaces.go

@@ -1,15 +0,0 @@
-package dns
-
-import (
-	"context"
-
-	"github.com/qdm12/dns/pkg/unbound"
-)
-
-type Configurator interface {
-	SetupFiles(ctx context.Context) error
-	MakeUnboundConf(settings unbound.Settings) (err error)
-	Start(ctx context.Context, verbosityDetailsLevel uint8) (
-		stdoutLines, stderrLines chan string, waitError chan error, err error)
-	Version(ctx context.Context) (version string, err error)
-}

internal/dns/logs.go

@@ -1,75 +0,0 @@
-package dns
-
-import (
-	"context"
-	"regexp"
-	"strings"
-
-	"github.com/qdm12/gluetun/internal/constants"
-)
-
-type logLevel uint8
-
-const (
-	levelDebug logLevel = iota
-	levelInfo
-	levelWarn
-	levelError
-)
-
-func (l *Loop) collectLines(ctx context.Context, done chan<- struct{},
-	stdout, stderr chan string) {
-	defer close(done)
-
-	var line string
-
-	for {
-		select {
-		case <-ctx.Done():
-			// Context should only be canceled after stdout and stderr are done
-			// being written to.
-			close(stdout)
-			close(stderr)
-			return
-		case line = <-stderr:
-		case line = <-stdout:
-		}
-
-		line, level := processLogLine(line)
-		switch level {
-		case levelDebug:
-			l.logger.Debug(line)
-		case levelInfo:
-			l.logger.Info(line)
-		case levelWarn:
-			l.logger.Warn(line)
-		case levelError:
-			l.logger.Error(line)
-		}
-	}
-}
-
-var unboundPrefix = regexp.MustCompile(`\[[0-9]{10}\] unbound\[[0-9]+:[0|1]\] `)
-
-func processLogLine(s string) (filtered string, level logLevel) {
-	prefix := unboundPrefix.FindString(s)
-	filtered = s[len(prefix):]
-	switch {
-	case strings.HasPrefix(filtered, "notice: "):
-		filtered = strings.TrimPrefix(filtered, "notice: ")
-		level = levelInfo
-	case strings.HasPrefix(filtered, "info: "):
-		filtered = strings.TrimPrefix(filtered, "info: ")
-		level = levelInfo
-	case strings.HasPrefix(filtered, "warn: "):
-		filtered = strings.TrimPrefix(filtered, "warn: ")
-		level = levelWarn
-	case strings.HasPrefix(filtered, "error: "):
-		filtered = strings.TrimPrefix(filtered, "error: ")
-		level = levelError
-	default:
-		level = levelInfo
-	}
-	filtered = constants.ColorUnbound().Sprintf(filtered)
-	return filtered, level
-}

internal/dns/logs_test.go

@@ -1,48 +0,0 @@
-package dns
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_processLogLine(t *testing.T) {
-	t.Parallel()
-	tests := map[string]struct {
-		s        string
-		filtered string
-		level    logLevel
-	}{
-		"empty string":  {"", "", levelInfo},
-		"random string": {"asdasqdb", "asdasqdb", levelInfo},
-		"unbound notice": {
-			"[1594595249] unbound[75:0] notice: init module 0: validator",
-			"init module 0: validator",
-			levelInfo},
-		"unbound info": {
-			"[1594595249] unbound[75:0] info: init module 0: validator",
-			"init module 0: validator",
-			levelInfo},
-		"unbound warn": {
-			"[1594595249] unbound[75:0] warn: init module 0: validator",
-			"init module 0: validator",
-			levelWarn},
-		"unbound error": {
-			"[1594595249] unbound[75:0] error: init module 0: validator",
-			"init module 0: validator",
-			levelError},
-		"unbound unknown": {
-			"[1594595249] unbound[75:0] BLA: init module 0: validator",
-			"BLA: init module 0: validator",
-			levelInfo},
-	}
-	for name, tc := range tests {
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			filtered, level := processLogLine(tc.s)
-			assert.Equal(t, tc.filtered, filtered)
-			assert.Equal(t, tc.level, level)
-		})
-	}
-}

internal/dns/loop.go

@@ -2,10 +2,12 @@ package dns
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
-	"github.com/qdm12/dns/pkg/blacklist"
+	"github.com/qdm12/dns/v2/pkg/dot"
+	"github.com/qdm12/dns/v2/pkg/middlewares/filter/mapfilter"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/dns/state"
@@ -16,9 +18,9 @@ import (
 type Loop struct {
 	statusManager *loopstate.State
 	state         *state.State
-	conf          Configurator
+	server        *dot.Server
+	filter        *mapfilter.Filter
 	resolvConf    string
-	blockBuilder  blacklist.Builder
 	client        *http.Client
 	logger        Logger
 	userTrigger   bool
@@ -34,8 +36,8 @@ type Loop struct {
 
 const defaultBackoffTime = 10 * time.Second
 
-func NewLoop(conf Configurator, settings settings.DNS,
-	client *http.Client, logger Logger) *Loop {
+func NewLoop(settings settings.DNS,
+	client *http.Client, logger Logger) (loop *Loop, err error) {
 	start := make(chan struct{})
 	running := make(chan models.LoopStatus)
 	stop := make(chan struct{})
@@ -45,12 +47,17 @@ func NewLoop(conf Configurator, settings settings.DNS,
 	statusManager := loopstate.New(constants.Stopped, start, running, stop, stopped)
 	state := state.New(statusManager, settings, updateTicker)
 
+	filter, err := mapfilter.New(mapfilter.Settings{})
+	if err != nil {
+		return nil, fmt.Errorf("creating map filter: %w", err)
+	}
+
 	return &Loop{
 		statusManager: statusManager,
 		state:         state,
-		conf:          conf,
+		server:        nil,
+		filter:        filter,
 		resolvConf:    "/etc/resolv.conf",
-		blockBuilder:  blacklist.NewBuilder(client),
 		client:        client,
 		logger:        logger,
 		userTrigger:   true,
@@ -62,7 +69,7 @@ func NewLoop(conf Configurator, settings settings.DNS,
 		backoffTime:   defaultBackoffTime,
 		timeNow:       time.Now,
 		timeSince:     time.Since,
-	}
+	}, nil
 }
 
 func (l *Loop) logAndWait(ctx context.Context, err error) {

internal/dns/plaintext.go

@@ -2,36 +2,22 @@ package dns
 
 import (
 	"net/netip"
+	"time"
 
-	"github.com/qdm12/dns/pkg/nameserver"
+	"github.com/qdm12/dns/v2/pkg/nameserver"
 )
 
 func (l *Loop) useUnencryptedDNS(fallback bool) {
 	settings := l.GetSettings()
 
 	// Try with user provided plaintext ip address
-	// if it's not 127.0.0.1 (default for DoT)
-	targetIP := settings.ServerAddress
-	if targetIP.Compare(netip.AddrFrom4([4]byte{127, 0, 0, 1})) != 0 {
-		if fallback {
-			l.logger.Info("falling back on plaintext DNS at address " + targetIP.String())
-		} else {
-			l.logger.Info("using plaintext DNS at address " + targetIP.String())
-		}
-		nameserver.UseDNSInternally(targetIP.AsSlice())
-		const keepNameserver = false
-		err := nameserver.UseDNSSystemWide(l.resolvConf, targetIP.AsSlice(), keepNameserver)
-		if err != nil {
-			l.logger.Error(err.Error())
-		}
-		return
-	}
-
-	// Use first plaintext DNS IPv4 address
-	targetIP, err := settings.DoT.Unbound.GetFirstPlaintextIPv4()
-	if err != nil {
-		// Unbound should always have a default provider
-		panic(err)
+	// if it's not 127.0.0.1 (default for DoT), otherwise
+	// use the first DoT provider ipv4 address found.
+	var targetIP netip.Addr
+	if settings.ServerAddress.Compare(netip.AddrFrom4([4]byte{127, 0, 0, 1})) != 0 {
+		targetIP = settings.ServerAddress
+	} else {
+		targetIP = settings.DoT.GetFirstPlaintextIPv4()
 	}
 
 	if fallback {
@@ -39,9 +25,19 @@ func (l *Loop) useUnencryptedDNS(fallback bool) {
 	} else {
 		l.logger.Info("using plaintext DNS at address " + targetIP.String())
 	}
-	nameserver.UseDNSInternally(targetIP.AsSlice())
-	const keepNameserver = false
-	err = nameserver.UseDNSSystemWide(l.resolvConf, targetIP.AsSlice(), keepNameserver)
+
+	const dialTimeout = 3 * time.Second
+	settingsInternalDNS := nameserver.SettingsInternalDNS{
+		IP:      targetIP,
+		Timeout: dialTimeout,
+	}
+	nameserver.UseDNSInternally(settingsInternalDNS)
+
+	settingsSystemWide := nameserver.SettingsSystemDNS{
+		IP:         targetIP,
+		ResolvPath: l.resolvConf,
+	}
+	err := nameserver.UseDNSSystemWide(settingsSystemWide)
 	if err != nil {
 		l.logger.Error(err.Error())
 	}

internal/dns/run.go

@@ -26,16 +26,14 @@ func (l *Loop) Run(ctx context.Context, done chan<- struct{}) {
 	}
 
 	for ctx.Err() == nil {
-		// Upper scope variables for Unbound only
+		// Upper scope variables for the DNS over TLS server only
 		// Their values are to be used if DOT=off
-		waitError := make(chan error)
-		unboundCancel := func() { waitError <- nil }
-		closeStreams := func() {}
+		var runError <-chan error
 
 		settings := l.GetSettings()
 		for !*settings.KeepNameserver && *settings.DoT.Enabled {
 			var err error
-			unboundCancel, waitError, closeStreams, err = l.setupUnbound(ctx)
+			runError, err = l.setupServer(ctx)
 			if err == nil {
 				l.backoffTime = defaultBackoffTime
 				l.logger.Info("ready")
@@ -49,11 +47,12 @@ func (l *Loop) Run(ctx context.Context, done chan<- struct{}) {
 				return
 			}
 
-			if !errors.Is(err, errUpdateFiles) {
+			if !errors.Is(err, errUpdateBlockLists) {
 				const fallback = true
 				l.useUnencryptedDNS(fallback)
 			}
 			l.logAndWait(ctx, err)
+			settings = l.GetSettings()
 		}
 
 		settings = l.GetSettings()
@@ -64,40 +63,44 @@ func (l *Loop) Run(ctx context.Context, done chan<- struct{}) {
 
 		l.userTrigger = false
 
-		stayHere := true
-		for stayHere {
-			select {
-			case <-ctx.Done():
-				unboundCancel()
-				<-waitError
-				close(waitError)
-				closeStreams()
-				return
-			case <-l.stop:
-				l.userTrigger = true
-				l.logger.Info("stopping")
-				const fallback = false
-				l.useUnencryptedDNS(fallback)
-				unboundCancel()
-				<-waitError
-				// do not close waitError or the waitError
-				// select case will trigger
-				closeStreams()
-				l.stopped <- struct{}{}
-			case <-l.start:
-				l.userTrigger = true
-				l.logger.Info("starting")
-				stayHere = false
-			case err := <-waitError: // unexpected error
-				closeStreams()
-
-				unboundCancel()
-				l.statusManager.SetStatus(constants.Crashed)
-				const fallback = true
-				l.useUnencryptedDNS(fallback)
-				l.logAndWait(ctx, err)
-				stayHere = false
-			}
+		exitLoop := l.runWait(ctx, runError)
+		if exitLoop {
+			return
 		}
 	}
 }
+
+func (l *Loop) runWait(ctx context.Context, runError <-chan error) (exitLoop bool) {
+	for {
+		select {
+		case <-ctx.Done():
+			l.stopServer()
+			// TODO revert OS and Go nameserver when exiting
+			return true
+		case <-l.stop:
+			l.userTrigger = true
+			l.logger.Info("stopping")
+			const fallback = false
+			l.useUnencryptedDNS(fallback)
+			l.stopServer()
+			l.stopped <- struct{}{}
+		case <-l.start:
+			l.userTrigger = true
+			l.logger.Info("starting")
+			return false
+		case err := <-runError: // unexpected error
+			l.statusManager.SetStatus(constants.Crashed)
+			const fallback = true
+			l.useUnencryptedDNS(fallback)
+			l.logAndWait(ctx, err)
+			return false
+		}
+	}
+}
+
+func (l *Loop) stopServer() {
+	stopErr := l.server.Stop()
+	if stopErr != nil {
+		l.logger.Error("stopping DoT server: " + stopErr.Error())
+	}
+}

internal/dns/settings.go

@@ -2,7 +2,14 @@ package dns
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/qdm12/dns/v2/pkg/dot"
+	cachemiddleware "github.com/qdm12/dns/v2/pkg/middlewares/cache"
+	"github.com/qdm12/dns/v2/pkg/middlewares/cache/lru"
+	filtermiddleware "github.com/qdm12/dns/v2/pkg/middlewares/filter"
+	"github.com/qdm12/dns/v2/pkg/middlewares/filter/mapfilter"
+	"github.com/qdm12/dns/v2/pkg/provider"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
 )
 
@@ -12,3 +19,55 @@ func (l *Loop) SetSettings(ctx context.Context, settings settings.DNS) (
 	outcome string) {
 	return l.state.SetSettings(ctx, settings)
 }
+
+func buildDoTSettings(settings settings.DNS,
+	filter *mapfilter.Filter, logger Logger) (
+	dotSettings dot.ServerSettings, err error) {
+	var middlewares []dot.Middleware
+
+	if *settings.DoT.Caching {
+		lruCache, err := lru.New(lru.Settings{})
+		if err != nil {
+			return dot.ServerSettings{}, fmt.Errorf("creating LRU cache: %w", err)
+		}
+		cacheMiddleware, err := cachemiddleware.New(cachemiddleware.Settings{
+			Cache: lruCache,
+		})
+		if err != nil {
+			return dot.ServerSettings{}, fmt.Errorf("creating cache middleware: %w", err)
+		}
+		middlewares = append(middlewares, cacheMiddleware)
+	}
+
+	filterMiddleware, err := filtermiddleware.New(filtermiddleware.Settings{
+		Filter: filter,
+	})
+	if err != nil {
+		return dot.ServerSettings{}, fmt.Errorf("creating filter middleware: %w", err)
+	}
+	middlewares = append(middlewares, filterMiddleware)
+
+	providersData := provider.NewProviders()
+	providers := make([]provider.Provider, len(settings.DoT.Providers))
+	for i := range settings.DoT.Providers {
+		var err error
+		providers[i], err = providersData.Get(settings.DoT.Providers[i])
+		if err != nil {
+			panic(err) // this should already had been checked
+		}
+	}
+
+	ipVersion := "ipv4"
+	if *settings.DoT.IPv6 {
+		ipVersion = "ipv6"
+	}
+	return dot.ServerSettings{
+		Resolver: dot.ResolverSettings{
+			UpstreamResolvers: providers,
+			IPVersion:         ipVersion,
+			Warner:            logger,
+		},
+		Middlewares: middlewares,
+		Logger:      logger,
+	}, nil
+}

internal/dns/setup.go

@@ -4,59 +4,55 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 
-	"github.com/qdm12/dns/pkg/check"
-	"github.com/qdm12/dns/pkg/nameserver"
+	"github.com/qdm12/dns/v2/pkg/check"
+	"github.com/qdm12/dns/v2/pkg/dot"
+	"github.com/qdm12/dns/v2/pkg/nameserver"
 )
 
-var errUpdateFiles = errors.New("cannot update files")
+var errUpdateBlockLists = errors.New("cannot update filter block lists")
 
-// Returning cancel == nil signals we want to re-run setupUnbound
-// Returning err == errUpdateFiles signals we should not fall back
-// on the plaintext DNS as DOT is still up and running.
-func (l *Loop) setupUnbound(ctx context.Context) (
-	cancel context.CancelFunc, waitError chan error, closeStreams func(), err error) {
+func (l *Loop) setupServer(ctx context.Context) (runError <-chan error, err error) {
 	err = l.updateFiles(ctx)
 	if err != nil {
-		return nil, nil, nil,
-			fmt.Errorf("%w: %s", errUpdateFiles, err)
+		return nil, fmt.Errorf("%w: %w", errUpdateBlockLists, err)
 	}
 
 	settings := l.GetSettings()
 
-	unboundCtx, cancel := context.WithCancel(context.Background())
-	stdoutLines, stderrLines, waitError, err := l.conf.Start(unboundCtx,
-		*settings.DoT.Unbound.VerbosityDetailsLevel)
+	dotSettings, err := buildDoTSettings(settings, l.filter, l.logger)
 	if err != nil {
-		cancel()
-		return nil, nil, nil, err
+		return nil, fmt.Errorf("building DoT settings: %w", err)
 	}
 
-	linesCollectionCtx, linesCollectionCancel := context.WithCancel(context.Background())
-	lineCollectionDone := make(chan struct{})
-	go l.collectLines(linesCollectionCtx, lineCollectionDone,
-		stdoutLines, stderrLines)
-	closeStreams = func() {
-		linesCollectionCancel()
-		<-lineCollectionDone
+	server, err := dot.NewServer(dotSettings)
+	if err != nil {
+		return nil, fmt.Errorf("creating DoT server: %w", err)
 	}
 
-	// use Unbound
-	nameserver.UseDNSInternally(settings.ServerAddress.AsSlice())
-	err = nameserver.UseDNSSystemWide(l.resolvConf, settings.ServerAddress.AsSlice(),
-		*settings.KeepNameserver)
+	runError, err = server.Start()
+	if err != nil {
+		return nil, fmt.Errorf("starting server: %w", err)
+	}
+	l.server = server
+
+	// use internal DNS server
+	nameserver.UseDNSInternally(nameserver.SettingsInternalDNS{
+		IP: settings.ServerAddress,
+	})
+	err = nameserver.UseDNSSystemWide(nameserver.SettingsSystemDNS{
+		IP:         settings.ServerAddress,
+		ResolvPath: l.resolvConf,
+	})
 	if err != nil {
 		l.logger.Error(err.Error())
 	}
 
-	if err := check.WaitForDNS(ctx, net.DefaultResolver); err != nil {
-		cancel()
-		<-waitError
-		close(waitError)
-		closeStreams()
-		return nil, nil, nil, err
+	err = check.WaitForDNS(ctx, check.Settings{})
+	if err != nil {
+		l.stopServer()
+		return nil, err
 	}
 
-	return cancel, waitError, closeStreams, nil
+	return runError, nil
 }

internal/dns/ticker.go

@@ -34,7 +34,7 @@ func (l *Loop) RunRestartTicker(ctx context.Context, done chan<- struct{}) {
 				if err := l.updateFiles(ctx); err != nil {
 					l.statusManager.SetStatus(constants.Crashed)
 					l.logger.Error(err.Error())
-					l.logger.Warn("skipping Unbound restart due to failed files update")
+					l.logger.Warn("skipping DNS server restart due to failed files update")
 					continue
 				}
 			}

internal/dns/update.go

@@ -1,35 +1,46 @@
 package dns
 
-import "context"
+import (
+	"context"
+	"fmt"
+
+	"github.com/qdm12/dns/v2/pkg/blockbuilder"
+	"github.com/qdm12/dns/v2/pkg/middlewares/filter/update"
+)
 
 func (l *Loop) updateFiles(ctx context.Context) (err error) {
-	l.logger.Info("downloading DNS over TLS cryptographic files")
-	if err := l.conf.SetupFiles(ctx); err != nil {
-		return err
-	}
 	settings := l.GetSettings()
 
-	unboundSettings, err := settings.DoT.Unbound.ToUnboundFormat()
-	if err != nil {
-		return err
-	}
-
 	l.logger.Info("downloading hostnames and IP block lists")
-	blacklistSettings, err := settings.DoT.Blacklist.ToBlacklistFormat()
+	blacklistSettings := settings.DoT.Blacklist.ToBlockBuilderSettings(l.client)
+
+	blockBuilder, err := blockbuilder.New(blacklistSettings)
+	if err != nil {
+		return fmt.Errorf("creating block builder: %w", err)
+	}
+
+	result := blockBuilder.BuildAll(ctx)
+	for _, resultErr := range result.Errors {
+		if err != nil {
+			err = fmt.Errorf("%w, %w", err, resultErr)
+			continue
+		}
+		err = resultErr
+	}
+
 	if err != nil {
 		return err
 	}
 
-	blockedHostnames, blockedIPs, blockedIPPrefixes, errs :=
-		l.blockBuilder.All(ctx, blacklistSettings)
-	for _, err := range errs {
-		l.logger.Warn(err.Error())
+	updateSettings := update.Settings{
+		IPs:        result.BlockedIPs,
+		IPPrefixes: result.BlockedIPPrefixes,
+	}
+	updateSettings.BlockHostnames(result.BlockedHostnames)
+	err = l.filter.Update(updateSettings)
+	if err != nil {
+		return fmt.Errorf("updating filter: %w", err)
 	}
 
-	// TODO change to BlockHostnames() when migrating to qdm12/dns v2
-	unboundSettings.Blacklist.FqdnHostnames = blockedHostnames
-	unboundSettings.Blacklist.IPs = blockedIPs
-	unboundSettings.Blacklist.IPPrefixes = blockedIPPrefixes
-
-	return l.conf.MakeUnboundConf(unboundSettings)
+	return nil
 }