From 25acbf8501f747eb67332f9f5dd3e36e25e8be68 Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Wed, 30 Dec 2020 17:22:54 +0000 Subject: [PATCH] Feature: Increasing backoff time for crashes - Fix #247 --- internal/dns/loop.go | 12 +++++++++--- internal/httpproxy/loop.go | 1 + internal/openvpn/loop.go | 11 ++++++++--- internal/publicip/loop.go | 11 ++++++++--- internal/shadowsocks/loop.go | 17 ++++++++++++++--- internal/updater/loop.go | 11 ++++++++--- 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/internal/dns/loop.go b/internal/dns/loop.go index cdbdbdc9..05bf411f 100644 --- a/internal/dns/loop.go +++ b/internal/dns/loop.go @@ -36,10 +36,13 @@ type looper struct { stop chan struct{} stopped chan struct{} updateTicker chan struct{} + backoffTime time.Duration timeNow func() time.Time timeSince func(time.Time) time.Duration } +const defaultBackoffTime = 10 * time.Second + func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger, streamMerger command.StreamMerger, username string, puid, pgid int) Looper { return &looper{ @@ -58,6 +61,7 @@ func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger, stop: make(chan struct{}), stopped: make(chan struct{}), updateTicker: make(chan struct{}), + backoffTime: defaultBackoffTime, timeNow: time.Now, timeSince: time.Since, } @@ -65,9 +69,9 @@ func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger, func (l *looper) logAndWait(ctx context.Context, err error) { l.logger.Warn(err) - l.logger.Info("attempting restart in 10 seconds") - const waitDuration = 10 * time.Second - timer := time.NewTimer(waitDuration) + l.logger.Info("attempting restart in %s", l.backoffTime) + timer := time.NewTimer(l.backoffTime) + l.backoffTime *= 2 select { case <-timer.C: case <-ctx.Done(): @@ -101,6 +105,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup, signalDNSReady fun } crashed := false + l.backoffTime = defaultBackoffTime for ctx.Err() == nil { settings := l.GetSettings() @@ -150,6 +155,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup, signalDNSReady fun l.running <- constants.Running crashed = false } else { + l.backoffTime = defaultBackoffTime l.state.setStatusWithLock(constants.Running) } signalDNSReady() diff --git a/internal/httpproxy/loop.go b/internal/httpproxy/loop.go index 4789f4bf..dec31ba8 100644 --- a/internal/httpproxy/loop.go +++ b/internal/httpproxy/loop.go @@ -105,6 +105,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { runCtx, runCancel := context.WithCancel(context.Background()) runWg := &sync.WaitGroup{} runWg.Add(1) + // TODO crashed channel go server.Run(runCtx, runWg) stayHere := true diff --git a/internal/openvpn/loop.go b/internal/openvpn/loop.go index e574933a..6fa35169 100644 --- a/internal/openvpn/loop.go +++ b/internal/openvpn/loop.go @@ -54,8 +54,11 @@ type looper struct { start chan struct{} portForwardSignals chan net.IP crashed bool + backoffTime time.Duration } +const defaultBackoffTime = 15 * time.Second + func NewLooper(settings settings.OpenVPN, username string, puid, pgid int, allServers models.AllServers, conf Configurator, fw firewall.Configurator, routing routing.Routing, @@ -84,6 +87,7 @@ func NewLooper(settings settings.OpenVPN, stop: make(chan struct{}), stopped: make(chan struct{}), portForwardSignals: make(chan net.IP), + backoffTime: defaultBackoffTime, } } @@ -179,6 +183,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { if l.crashed { l.crashed = false + l.backoffTime = defaultBackoffTime l.state.setStatusWithLock(constants.Running) } else { l.running <- constants.Running @@ -216,9 +221,9 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { func (l *looper) logAndWait(ctx context.Context, err error) { l.logger.Error(err) - const waitTime = 30 * time.Second - l.logger.Info("retrying in %s", waitTime) - timer := time.NewTimer(waitTime) + l.logger.Info("retrying in %s", l.backoffTime) + timer := time.NewTimer(l.backoffTime) + l.backoffTime *= 2 select { case <-timer.C: case <-ctx.Done(): diff --git a/internal/publicip/loop.go b/internal/publicip/loop.go index b803be35..e86fac7d 100644 --- a/internal/publicip/loop.go +++ b/internal/publicip/loop.go @@ -40,11 +40,14 @@ type looper struct { stop chan struct{} stopped chan struct{} updateTicker chan struct{} + backoffTime time.Duration // Mock functions timeNow func() time.Time timeSince func(time.Time) time.Duration } +const defaultBackoffTime = 5 * time.Second + func NewLooper(client *http.Client, logger logging.Logger, settings settings.PublicIP, puid, pgid int, os os.OS) Looper { @@ -64,6 +67,7 @@ func NewLooper(client *http.Client, logger logging.Logger, stop: make(chan struct{}), stopped: make(chan struct{}), updateTicker: make(chan struct{}), + backoffTime: defaultBackoffTime, timeNow: time.Now, timeSince: time.Since, } @@ -71,9 +75,9 @@ func NewLooper(client *http.Client, logger logging.Logger, func (l *looper) logAndWait(ctx context.Context, err error) { l.logger.Error(err) - const waitTime = 5 * time.Second - l.logger.Info("retrying in %s", waitTime) - timer := time.NewTimer(waitTime) + l.logger.Info("retrying in %s", l.backoffTime) + timer := time.NewTimer(l.backoffTime) + l.backoffTime *= 2 select { case <-timer.C: case <-ctx.Done(): @@ -116,6 +120,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { l.running <- constants.Running crashed = false } else { + l.backoffTime = defaultBackoffTime l.state.setStatusWithLock(constants.Running) } diff --git a/internal/shadowsocks/loop.go b/internal/shadowsocks/loop.go index aaf9b5b3..defbf2c6 100644 --- a/internal/shadowsocks/loop.go +++ b/internal/shadowsocks/loop.go @@ -28,13 +28,14 @@ type looper struct { restart chan struct{} start chan struct{} stop chan struct{} + backoffTime time.Duration } func (l *looper) logAndWait(ctx context.Context, err error) { l.logger.Error(err) - const waitTime = time.Minute - l.logger.Info("retrying in %s", waitTime) - timer := time.NewTimer(waitTime) + l.logger.Info("retrying in %s", l.backoffTime) + timer := time.NewTimer(l.backoffTime) + l.backoffTime *= 2 select { case <-timer.C: case <-ctx.Done(): @@ -44,6 +45,8 @@ func (l *looper) logAndWait(ctx context.Context, err error) { } } +const defaultBackoffTime = 10 * time.Second + func NewLooper(settings settings.ShadowSocks, logger logging.Logger, defaultInterface string) Looper { return &looper{ settings: settings, @@ -52,6 +55,7 @@ func NewLooper(settings settings.ShadowSocks, logger logging.Logger, defaultInte restart: make(chan struct{}), start: make(chan struct{}), stop: make(chan struct{}), + backoffTime: defaultBackoffTime, } } @@ -136,6 +140,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { continue } + isStableTimer := time.NewTimer(time.Second) + stayHere := true for stayHere { select { @@ -145,6 +151,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { <-waitError close(waitError) return + case <-isStableTimer.C: + l.backoffTime = defaultBackoffTime case <-l.restart: // triggered restart l.logger.Info("restarting") shadowsocksCancel() @@ -167,5 +175,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { } } shadowsocksCancel() // repetition for linter only + if !isStableTimer.Stop() { + <-isStableTimer.C + } } } diff --git a/internal/updater/loop.go b/internal/updater/loop.go index 70040714..84f61248 100644 --- a/internal/updater/loop.go +++ b/internal/updater/loop.go @@ -36,11 +36,14 @@ type looper struct { stop chan struct{} stopped chan struct{} updateTicker chan struct{} + backoffTime time.Duration // Mock functions timeNow func() time.Time timeSince func(time.Time) time.Duration } +const defaultBackoffTime = 5 * time.Second + func NewLooper(settings settings.Updater, currentServers models.AllServers, storage storage.Storage, setAllServers func(allServers models.AllServers), client *http.Client, logger logging.Logger) Looper { @@ -61,14 +64,15 @@ func NewLooper(settings settings.Updater, currentServers models.AllServers, updateTicker: make(chan struct{}), timeNow: time.Now, timeSince: time.Since, + backoffTime: defaultBackoffTime, } } func (l *looper) logAndWait(ctx context.Context, err error) { l.logger.Error(err) - const waitTime = 5 * time.Minute - l.logger.Info("retrying in %s", waitTime) - timer := time.NewTimer(waitTime) + l.logger.Info("retrying in %s", l.backoffTime) + timer := time.NewTimer(l.backoffTime) + l.backoffTime *= 2 select { case <-timer.C: case <-ctx.Done(): @@ -108,6 +112,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) { l.running <- constants.Running crashed = false } else { + l.backoffTime = defaultBackoffTime l.state.setStatusWithLock(constants.Running) }