Always ensure iptables masquerade rules are installed

main.go

@@ -285,7 +285,7 @@ func main() {
 
 	// Set up ipMasq if needed
 	if opts.ipMasq {
-		setupIPMasq(config, bn)
+		go setupIPMasq(config, bn)
 	}
 
 	if err := WriteSubnetFile(opts.subnetFile, config.Network, opts.ipMasq, bn); err != nil {
@@ -556,20 +556,23 @@ func mustRunHealthz() {
 func setupIPMasq(config *subnet.Config, bn backend.Network) {
 	ipt, err := iptables.New()
 	if err != nil {
+		// if we can't find iptables, give up and return
 		log.Errorf("Failed to set up IP Masquerade. iptables was not found: %v", err)
 		return
 	}
-	err = network.SetupIPMasq(ipt, config.Network, bn.Lease())
-	if err != nil {
-		// Continue, even though it failed.
-		log.Errorf("Failed to set up IP Masquerade: %v", err)
-	}
-
 	defer func() {
 		if err := network.TeardownIPMasq(ipt, config.Network, bn.Lease()); err != nil {
 			log.Errorf("Failed to tear down IP Masquerade: %v", err)
 		}
 	}()
+	for {
+		// Ensure that all the rules exist every 5 seconds
+		if err := network.EnsureIPMasq(ipt, config.Network, bn.Lease()); err != nil {
+			log.Errorf("Failed to ensure IP Masquerade: %v", err)
+		}
+		time.Sleep(5 * time.Second)
+	}
+
 }
 
 func ReadSubnetFromSubnetFile(path string) ip.IP4Net {

network/ipmasq.go

@@ -27,6 +27,7 @@ import (
 type IPTablesRules interface {
 	AppendUnique(table string, chain string, rulespec ...string) error
 	Delete(table string, chain string, rulespec ...string) error
+	Exists(table string, chain string, rulespec ...string) (bool, error)
 }
 
 func rules(ipn ip.IP4Net, lease *subnet.Lease) [][]string {
@@ -45,6 +46,42 @@ func rules(ipn ip.IP4Net, lease *subnet.Lease) [][]string {
 	}
 }
 
+func ipMasqRulesExist(ipt IPTablesRules, ipn ip.IP4Net, lease *subnet.Lease) (bool, error) {
+	for _, rule := range rules(ipn, lease) {
+		log.Info("Adding iptables rule: ", strings.Join(rule, " "))
+		exists, err := ipt.Exists("nat", "POSTROUTING", rule...)
+		if err != nil {
+			// this shouldn't ever happen
+			return false, fmt.Errorf("failed to check rule existence", err)
+		}
+		if !exists {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func EnsureIPMasq(ipt IPTablesRules, ipn ip.IP4Net, lease *subnet.Lease) error {
+	exists, err := ipMasqRulesExist(ipt, ipn, lease)
+	if err != nil {
+		return fmt.Errorf("Error checking rule existence: %v", err)
+	}
+	if exists {
+		// if all the rules already exist, no need to do anything
+		return nil
+	}
+	// Otherwise, teardown all the rules and set them up again
+	// We do this because the order of the rules is important
+	if err = TeardownIPMasq(ipt, ipn, lease); err != nil {
+		return fmt.Errorf("Error tearing down rules: %v", err)
+	}
+	if err = SetupIPMasq(ipt, ipn, lease); err != nil {
+		return fmt.Errorf("Error setting up rules: %v", err)
+	}
+	return nil
+}
+
 func SetupIPMasq(ipt IPTablesRules, ipn ip.IP4Net, lease *subnet.Lease) error {
 	for _, rule := range rules(ipn, lease) {
 		log.Info("Adding iptables rule: ", strings.Join(rule, " "))

network/ipmasq_test.go

@@ -56,6 +56,14 @@ func (mock *MockIPTables) Delete(table string, chain string, rulespec ...string)
 	return nil
 }
 
+func (mock *MockIPTables) Exists(table string, chain string, rulespec ...string) (bool, error) {
+	var ruleIndex = mock.ruleIndex(table, chain, rulespec)
+	if ruleIndex != -1 {
+		return true, nil
+	}
+	return false, nil
+}
+
 func (mock *MockIPTables) AppendUnique(table string, chain string, rulespec ...string) error {
 	var ruleIndex = mock.ruleIndex(table, chain, rulespec)
 	if ruleIndex == -1 {
@@ -75,3 +83,17 @@ func TestDeleteRules(t *testing.T) {
 		t.Errorf("Should be 0 rules, there are actually %d: %#v", len(ipt.rules), ipt.rules)
 	}
 }
+
+func TestEnsureRules(t *testing.T) {
+	// If any rules are missing, they should be all deleted and recreated in the correct order
+	ipt_correct := &MockIPTables{}
+	SetupIPMasq(ipt_correct, ip.IP4Net{}, lease())
+	// setup a mock instance where we delete some rules and run `EnsureIPMasq`
+	ipt_recreate := &MockIPTables{}
+	SetupIPMasq(ipt_recreate, ip.IP4Net{}, lease())
+	ipt_recreate.rules = ipt_recreate.rules[0:2]
+	EnsureIPMasq(ipt_recreate, ip.IP4Net{}, lease())
+	if !reflect.DeepEqual(ipt_recreate.rules, ipt_correct.rules) {
+		t.Errorf("iptables rules after EnsureIPMasq are incorrected. Expected: %#v, Actual: %#v", ipt_recreate.rules, ipt_correct.rules)
+	}
+}