BUG/MINOR: disable faulty backend config snippets

during reload, additional check is now made to ensure that runtime errors are handled too

Author: oktalz

pkg/annotations/cfgSnippet.go

@@ -281,6 +281,26 @@ func CheckBackendConfigSnippetError(configErr error, cfgDir string) (rerun bool,
 	return rerun, err
 }
 
+func CheckBackendConfigSnippetErrorOnReload(configErr error, cfgDir string) (rerun bool, err error) {
+	// No error ? no configsnippet to disable.
+	if configErr == nil {
+		return rerun, err
+	}
+	file, lineNumbers, err := processConfigurationError(configErr)
+	if err != nil {
+		return rerun, err
+	}
+	// Read contents from failed configuration file
+	file = filepath.Join(cfgDir, filepath.Base(file))
+	contents, err := os.ReadFile(file)
+	if err != nil {
+		return rerun, err
+	}
+
+	rerun = disableFaultyCfgSnippet(string(contents), lineNumbers)
+	return rerun, err
+}
+
 func RemoveBackendCfgSnippet(backend string) {
 	if cfgSnippet.backends == nil {
 		return

pkg/controller/controller.go

@@ -15,6 +15,7 @@
 package controller
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -102,7 +103,8 @@ func (c *HAProxyController) Start() {
 	c.initHandlers()
 	logger.Error(c.setupHAProxyRules())
 	logger.Error(os.Chdir(c.haproxy.Env.CfgDir))
-	logger.Panic(c.haproxy.Service("start"))
+	_, errStart := (c.haproxy.Service("start"))
+	logger.Panic(errStart)
 
 	c.SyncData()
 }
@@ -111,7 +113,8 @@ func (c *HAProxyController) Start() {
 func (c *HAProxyController) Stop() {
 	logger.Infof("Stopping Ingress Controller")
 	close(c.chShutdown)
-	logger.Error(c.haproxy.Service("stop"))
+	_, errStop := c.haproxy.Service("stop")
+	logger.Error(errStop)
 }
 
 // updateHAProxy is the control loop syncing HAProxy configuration
@@ -196,8 +199,32 @@ func (c *HAProxyController) updateHAProxy() {
 
 	if instance.NeedReload() {
 		fs.RunDelayedFuncs()
-		if err = c.haproxy.Service("reload"); err != nil {
+		var msg string
+		if msg, err = c.haproxy.Service("reload"); err != nil {
 			logger.Error(err)
+			errLines := strings.Split(msg, "\n")
+			msg := ""
+			// Extract only lines with [ALERT] prefix to reuse functions
+			for _, line := range errLines {
+				if strings.HasPrefix(line, "[ALERT]") {
+					msg += strings.TrimPrefix(line, "[ALERT]") + "\n"
+				}
+			}
+
+			c.prometheusMetricsManager.SetUnableSyncGauge()
+			rerun, errCfgSnippet := annotations.CheckBackendConfigSnippetErrorOnReload(errors.New(msg), c.haproxy.Env.CfgDir)
+			logger.Error(errCfgSnippet)
+			c.clean(true)
+			if rerun {
+				logger.Debug("disabling some config snippets because of errors")
+				// We need to replay all these resources.
+				c.store.SecretsProcessed = map[string]struct{}{}
+				c.store.BackendsProcessed = map[string]struct{}{}
+				c.updateHAProxy()
+				return
+			}
+			// If any error not from config snippet then pop the previous state of backends
+			logger.Error(c.haproxy.PopPreviousBackends())
 		} else {
 			logger.Info("HAProxy reloaded")
 		}

pkg/haproxy/process/direct-control.go

@@ -18,10 +18,10 @@ type directControl struct {
 	useAuxFile bool
 }
 
-func (d *directControl) Service(action string) (err error) {
+func (d *directControl) Service(action string) (msg string, err error) {
 	if d.OSArgs.Test {
 		logger.Infof("HAProxy would be %sed now", action)
-		return nil
+		return "", nil
 	}
 	var cmd *exec.Cmd
 	// if processErr is nil, process variable will automatically
@@ -35,33 +35,33 @@ func (d *directControl) Service(action string) (err error) {
 	case "start":
 		if processErr == nil {
 			logger.Error("haproxy is already running")
-			return nil
+			return "", nil
 		}
 		cmd = exec.Command(d.Env.Binary, "-W", "-S", masterSocketArg, "-f", d.Env.MainCFGFile)
 		if d.useAuxFile {
 			cmd = exec.Command(d.Env.Binary, "-W", "-S", masterSocketArg, "-f", d.Env.MainCFGFile, "-f", d.Env.AuxCFGFile)
 		}
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		return cmd.Run()
+		return "", cmd.Run()
 	case "stop":
 		if processErr != nil {
 			logger.Error("haproxy already stopped")
-			return processErr
+			return "", processErr
 		}
 		if err = process.Signal(syscall.SIGUSR1); err != nil {
-			return err
+			return "", err
 		}
 		_, err = process.Wait()
-		return err
+		return "", err
 	case "reload":
 		if processErr != nil {
 			logger.Errorf("haproxy is not running, trying to start it")
 			return d.Service("start")
 		}
-		return process.Signal(syscall.SIGUSR2)
+		return "", process.Signal(syscall.SIGUSR2)
 	default:
-		return fmt.Errorf("unknown command '%s'", action)
+		return "", fmt.Errorf("unknown command '%s'", action)
 	}
 }
 

pkg/haproxy/process/interface.go

@@ -17,7 +17,7 @@ var logger = utils.GetLogger()
 const MASTER_SOCKET_PATH = "/var/run/haproxy-master.sock" //nolint:stylecheck
 
 type Process interface {
-	Service(action string) (err error)
+	Service(action string) (msg string, err error)
 	UseAuxFile(useAuxFile bool)
 	SetAPI(api api.HAProxyClient)
 }
@@ -37,7 +37,7 @@ func New(env env.Env, osArgs utils.OSArgs, auxCfgFile string, api api.HAProxyCli
 		if _, err := os.Stat(auxCfgFile); err == nil {
 			p.UseAuxFile(true)
 		}
-		_ = p.Service("start")
+		_, _ = p.Service("start")
 	}
 	return p
 }

pkg/haproxy/process/pebble.go

@@ -40,20 +40,20 @@ func newPebbleControl(env env.Env, osArgs utils.OSArgs) *pebbleControl {
 	return &pb
 }
 
-func (d *pebbleControl) Service(action string) error {
+func (d *pebbleControl) Service(action string) (string, error) {
 	if d.OSArgs.Test {
 		logger.Infof("HAProxy would be %sed now", action)
-		return nil
+		return "", nil
 	}
 	var cmd *exec.Cmd
 
 	switch action {
 	case "start":
 		// no need to start it is up already (pebble)
-		return nil
+		return "", nil
 	case "stop":
 		// no need to stop it (pebble)
-		return nil
+		return "", nil
 	case "reload":
 		if d.masterSocketValid {
 			msg, err := d.masterSocket.Reload()
@@ -62,14 +62,14 @@ func (d *pebbleControl) Service(action string) error {
 			}
 			d.logger.Debug("Reload done")
 			d.logger.Debug(msg)
-			return err
+			return msg, err
 		}
 		cmd = exec.Command("pebble", "signal", "SIGUSR2", "haproxy")
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		return cmd.Run()
+		return "", cmd.Run()
 	default:
-		return fmt.Errorf("unknown command '%s'", action)
+		return "", fmt.Errorf("unknown command '%s'", action)
 	}
 }
 

pkg/haproxy/process/s6-overlay.go

@@ -42,36 +42,37 @@ func newS6Control(api api.HAProxyClient, env env.Env, osArgs utils.OSArgs) *s6Co
 	return &sc
 }
 
-func (d *s6Control) Service(action string) error {
+func (d *s6Control) Service(action string) (string, error) {
 	if d.OSArgs.Test {
 		logger.Infof("HAProxy would be %sed now", action)
-		return nil
+		return "", nil
 	}
 	var cmd *exec.Cmd
 
 	switch action {
 	case "start":
 		// no need to start it is up already (s6)
-		return nil
+		return "", nil
 	case "stop":
 		// no need to stop it (s6)
-		return nil
+		return "", nil
 	case "reload":
 		if d.masterSocketValid {
 			msg, err := d.masterSocket.Reload()
 			if err == nil {
 				d.logger.Debug(msg)
-				return nil
+				return "", nil
 			}
 			d.logger.Error(err)
+			return msg, err
 		}
 
 		cmd = exec.Command("s6-svc", "-2", "/run/service/haproxy")
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		return cmd.Run()
+		return "", cmd.Run()
 	default:
-		return fmt.Errorf("unknown command '%s'", action)
+		return "", fmt.Errorf("unknown command '%s'", action)
 	}
 }