Feature: Increasing backoff time for crashes

- Fix #247

internal/dns/loop.go

@@ -36,10 +36,13 @@ type looper struct {
 	stop         chan struct{}
 	stopped      chan struct{}
 	updateTicker chan struct{}
+	backoffTime  time.Duration
 	timeNow      func() time.Time
 	timeSince    func(time.Time) time.Duration
 }
 
+const defaultBackoffTime = 10 * time.Second
+
 func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger,
 	streamMerger command.StreamMerger, username string, puid, pgid int) Looper {
 	return &looper{
@@ -58,6 +61,7 @@ func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger,
 		stop:         make(chan struct{}),
 		stopped:      make(chan struct{}),
 		updateTicker: make(chan struct{}),
+		backoffTime:  defaultBackoffTime,
 		timeNow:      time.Now,
 		timeSince:    time.Since,
 	}
@@ -65,9 +69,9 @@ func NewLooper(conf Configurator, settings settings.DNS, logger logging.Logger,
 
 func (l *looper) logAndWait(ctx context.Context, err error) {
 	l.logger.Warn(err)
-	l.logger.Info("attempting restart in 10 seconds")
+	l.logger.Info("attempting restart in %s", l.backoffTime)
-	const waitDuration = 10 * time.Second
+	timer := time.NewTimer(l.backoffTime)
-	timer := time.NewTimer(waitDuration)
+	l.backoffTime *= 2
 	select {
 	case <-timer.C:
 	case <-ctx.Done():
@@ -101,6 +105,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup, signalDNSReady fun
 	}
 
 	crashed := false
+	l.backoffTime = defaultBackoffTime
 
 	for ctx.Err() == nil {
 		settings := l.GetSettings()
@@ -150,6 +155,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup, signalDNSReady fun
 			l.running <- constants.Running
 			crashed = false
 		} else {
+			l.backoffTime = defaultBackoffTime
 			l.state.setStatusWithLock(constants.Running)
 		}
 		signalDNSReady()

internal/httpproxy/loop.go

@@ -105,6 +105,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 		runCtx, runCancel := context.WithCancel(context.Background())
 		runWg := &sync.WaitGroup{}
 		runWg.Add(1)
+		// TODO crashed channel
 		go server.Run(runCtx, runWg)
 
 		stayHere := true

internal/openvpn/loop.go

@@ -54,8 +54,11 @@ type looper struct {
 	start              chan struct{}
 	portForwardSignals chan net.IP
 	crashed            bool
+	backoffTime        time.Duration
 }
 
+const defaultBackoffTime = 15 * time.Second
+
 func NewLooper(settings settings.OpenVPN,
 	username string, puid, pgid int, allServers models.AllServers,
 	conf Configurator, fw firewall.Configurator, routing routing.Routing,
@@ -84,6 +87,7 @@ func NewLooper(settings settings.OpenVPN,
 		stop:               make(chan struct{}),
 		stopped:            make(chan struct{}),
 		portForwardSignals: make(chan net.IP),
+		backoffTime:        defaultBackoffTime,
 	}
 }
 
@@ -179,6 +183,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 
 		if l.crashed {
 			l.crashed = false
+			l.backoffTime = defaultBackoffTime
 			l.state.setStatusWithLock(constants.Running)
 		} else {
 			l.running <- constants.Running
@@ -216,9 +221,9 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 
 func (l *looper) logAndWait(ctx context.Context, err error) {
 	l.logger.Error(err)
-	const waitTime = 30 * time.Second
+	l.logger.Info("retrying in %s", l.backoffTime)
-	l.logger.Info("retrying in %s", waitTime)
+	timer := time.NewTimer(l.backoffTime)
-	timer := time.NewTimer(waitTime)
+	l.backoffTime *= 2
 	select {
 	case <-timer.C:
 	case <-ctx.Done():

internal/publicip/loop.go

@@ -40,11 +40,14 @@ type looper struct {
 	stop         chan struct{}
 	stopped      chan struct{}
 	updateTicker chan struct{}
+	backoffTime  time.Duration
 	// Mock functions
 	timeNow   func() time.Time
 	timeSince func(time.Time) time.Duration
 }
 
+const defaultBackoffTime = 5 * time.Second
+
 func NewLooper(client *http.Client, logger logging.Logger,
 	settings settings.PublicIP, puid, pgid int,
 	os os.OS) Looper {
@@ -64,6 +67,7 @@ func NewLooper(client *http.Client, logger logging.Logger,
 		stop:         make(chan struct{}),
 		stopped:      make(chan struct{}),
 		updateTicker: make(chan struct{}),
+		backoffTime:  defaultBackoffTime,
 		timeNow:      time.Now,
 		timeSince:    time.Since,
 	}
@@ -71,9 +75,9 @@ func NewLooper(client *http.Client, logger logging.Logger,
 
 func (l *looper) logAndWait(ctx context.Context, err error) {
 	l.logger.Error(err)
-	const waitTime = 5 * time.Second
+	l.logger.Info("retrying in %s", l.backoffTime)
-	l.logger.Info("retrying in %s", waitTime)
+	timer := time.NewTimer(l.backoffTime)
-	timer := time.NewTimer(waitTime)
+	l.backoffTime *= 2
 	select {
 	case <-timer.C:
 	case <-ctx.Done():
@@ -116,6 +120,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 			l.running <- constants.Running
 			crashed = false
 		} else {
+			l.backoffTime = defaultBackoffTime
 			l.state.setStatusWithLock(constants.Running)
 		}
 

internal/shadowsocks/loop.go

@@ -28,13 +28,14 @@ type looper struct {
 	restart          chan struct{}
 	start            chan struct{}
 	stop             chan struct{}
+	backoffTime      time.Duration
 }
 
 func (l *looper) logAndWait(ctx context.Context, err error) {
 	l.logger.Error(err)
-	const waitTime = time.Minute
+	l.logger.Info("retrying in %s", l.backoffTime)
-	l.logger.Info("retrying in %s", waitTime)
+	timer := time.NewTimer(l.backoffTime)
-	timer := time.NewTimer(waitTime)
+	l.backoffTime *= 2
 	select {
 	case <-timer.C:
 	case <-ctx.Done():
@@ -44,6 +45,8 @@ func (l *looper) logAndWait(ctx context.Context, err error) {
 	}
 }
 
+const defaultBackoffTime = 10 * time.Second
+
 func NewLooper(settings settings.ShadowSocks, logger logging.Logger, defaultInterface string) Looper {
 	return &looper{
 		settings:         settings,
@@ -52,6 +55,7 @@ func NewLooper(settings settings.ShadowSocks, logger logging.Logger, defaultInte
 		restart:          make(chan struct{}),
 		start:            make(chan struct{}),
 		stop:             make(chan struct{}),
+		backoffTime:      defaultBackoffTime,
 	}
 }
 
@@ -136,6 +140,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 			continue
 		}
 
+		isStableTimer := time.NewTimer(time.Second)
+
 		stayHere := true
 		for stayHere {
 			select {
@@ -145,6 +151,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 				<-waitError
 				close(waitError)
 				return
+			case <-isStableTimer.C:
+				l.backoffTime = defaultBackoffTime
 			case <-l.restart: // triggered restart
 				l.logger.Info("restarting")
 				shadowsocksCancel()
@@ -167,5 +175,8 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 			}
 		}
 		shadowsocksCancel() // repetition for linter only
+		if !isStableTimer.Stop() {
+			<-isStableTimer.C
+		}
 	}
 }

internal/updater/loop.go

@@ -36,11 +36,14 @@ type looper struct {
 	stop         chan struct{}
 	stopped      chan struct{}
 	updateTicker chan struct{}
+	backoffTime  time.Duration
 	// Mock functions
 	timeNow   func() time.Time
 	timeSince func(time.Time) time.Duration
 }
 
+const defaultBackoffTime = 5 * time.Second
+
 func NewLooper(settings settings.Updater, currentServers models.AllServers,
 	storage storage.Storage, setAllServers func(allServers models.AllServers),
 	client *http.Client, logger logging.Logger) Looper {
@@ -61,14 +64,15 @@ func NewLooper(settings settings.Updater, currentServers models.AllServers,
 		updateTicker:  make(chan struct{}),
 		timeNow:       time.Now,
 		timeSince:     time.Since,
+		backoffTime:   defaultBackoffTime,
 	}
 }
 
 func (l *looper) logAndWait(ctx context.Context, err error) {
 	l.logger.Error(err)
-	const waitTime = 5 * time.Minute
+	l.logger.Info("retrying in %s", l.backoffTime)
-	l.logger.Info("retrying in %s", waitTime)
+	timer := time.NewTimer(l.backoffTime)
-	timer := time.NewTimer(waitTime)
+	l.backoffTime *= 2
 	select {
 	case <-timer.C:
 	case <-ctx.Done():
@@ -108,6 +112,7 @@ func (l *looper) Run(ctx context.Context, wg *sync.WaitGroup) {
 			l.running <- constants.Running
 			crashed = false
 		} else {
+			l.backoffTime = defaultBackoffTime
 			l.state.setStatusWithLock(constants.Running)
 		}