backend/extension: Introduce new extension backend

Dockerfile.amd64

@@ -4,6 +4,7 @@ MAINTAINER Tom Denham <tom@tigera.io>
 
 ENV FLANNEL_ARCH=amd64
 
+RUN apk add --no-cache iproute2 net-tools
 COPY dist/flanneld-$FLANNEL_ARCH /opt/bin/flanneld
 COPY dist/iptables-$FLANNEL_ARCH /usr/local/bin/iptables
 COPY dist/mk-docker-opts.sh /opt/bin/

Documentation/extension.md

@@ -0,0 +1,59 @@
+The `extension` backend provides an easy way for prototyping new backend types for flannel.
+
+It is _not_ recommended for production use, for example it doesn't have a built in retry mechanism.
+
+This backend has the following configuration
+* `Type` (string): `extension`
+* `PreStartupCommand`  (string): Command to run before allocating a network to this host
+    * The stdout of the process is captured and passed to the stdin of the SubnetAdd/Remove commands.
+* `PostStartupCommand`  (string): Command to run after allocating a network to this host
+    * The following environment variable is set
+            * SUBNET - The subnet of the remote host that was added.
+* `SubnetAddCommand`   (string): Command to run when a subnet is added
+    * stdin - The output from `PreStartupCommand` is passed in.
+    * The following environment variables are set
+        * SUBNET - The subnet of the remote host that was added.
+        * PUBLIC_IP - The public IP of the remote host.
+* `SubnetRemoveCommand`(string): Command to run when a subnet is removed
+    * stdin - The output from `PreStartupCommand` is passed in.
+      * The following environment variables are set
+          * SUBNET - The subnet of the remote host that was removed.
+          * PUBLIC_IP - The public IP of the remote host.
+
+All commands are run through the `sh` shell and are run with the same permissions as the flannel daemon.
+
+
+## Simple example (host-gw)
+To replicate the functionality of the host-gw plugin, there's no need for a startup command.
+
+The backend just needs to manage the route to subnets when they are added or removed.
+
+An example
+```json
+{
+  "Network": "10.0.0.0/16",
+  "Backend": {
+    "Type": "extension",
+    "SubnetAddCommand": "ip route add $SUBNET via $PUBLIC_IP",
+    "SubnetRemoveCommand": "ip route del $SUBNET via $PUBLIC_IP"
+  }
+}
+```
+
+
+## Complex example (vxlan)
+VXLAN is more complex. It needs to store the MAC address of the vxlan device when it's created and to make it available to the flannel daemon running on other hosts.
+The address of the vxlan device also needs to be set _after_ the subnet has been allocated.
+
+An example
+```json
+{
+  "Network": "10.50.0.0/16",
+  "Backend": {
+    "Type": "extension",
+    "PreStartupCommand": "export VNI=1; export IF_NAME=flannel-vxlan; ip link del $IF_NAME 2>/dev/null; ip link add $IF_NAME type vxlan id $VNI dstport 8472 && cat /sys/class/net/$IF_NAME/address",
+    "PostStartupCommand": "export IF_NAME=flannel-vxlan; export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; ip addr add $SUBNET_IP/32 dev $IF_NAME && ip link set $IF_NAME up",
+    "SubnetAddCommand": "export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; export IF_NAME=flannel-vxlan; read VTEP; ip route add $SUBNET nexthop via $SUBNET_IP dev $IF_NAME onlink && arp -s $SUBNET_IP $VTEP dev $IF_NAME && bridge fdb add $VTEP dev $IF_NAME self dst $PUBLIC_IP"
+  }
+}
+```

backend/extension/extension.go

@@ -0,0 +1,150 @@
+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extension
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"os/exec"
+
+	"encoding/json"
+
+	log "github.com/golang/glog"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+	"golang.org/x/net/context"
+)
+
+func init() {
+	backend.Register("extension", New)
+}
+
+type ExtensionBackend struct {
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+	networks map[string]*network
+}
+
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	be := &ExtensionBackend{
+		sm:       sm,
+		extIface: extIface,
+		networks: make(map[string]*network),
+	}
+
+	return be, nil
+}
+
+func (_ *ExtensionBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *ExtensionBackend) RegisterNetwork(ctx context.Context, config *subnet.Config) (backend.Network, error) {
+	n := &network{
+		extIface: be.extIface,
+		sm:       be.sm,
+	}
+
+	// Parse out configuration
+	if len(config.Backend) > 0 {
+		cfg := struct {
+			PreStartupCommand   string
+			PostStartupCommand  string
+			SubnetAddCommand    string
+			SubnetRemoveCommand string
+		}{}
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
+			return nil, fmt.Errorf("error decoding backend config: %v", err)
+		}
+		n.preStartupCommand = cfg.PreStartupCommand
+		n.postStartupCommand = cfg.PostStartupCommand
+		n.subnetAddCommand = cfg.SubnetAddCommand
+		n.subnetRemoveCommand = cfg.SubnetRemoveCommand
+	}
+
+	data := []byte{}
+	if len(n.preStartupCommand) > 0 {
+		cmd_output, err := runCmd([]string{}, "", "sh", "-c", n.preStartupCommand)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run command: %s Err: %v Output: %s", n.preStartupCommand, err, cmd_output)
+		} else {
+			log.Infof("Ran command: %s\n Output: %s", n.preStartupCommand, cmd_output)
+		}
+
+		data, err = json.Marshal(cmd_output)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		log.Infof("No pre startup command configured - skipping")
+	}
+
+	attrs := subnet.LeaseAttrs{
+		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
+		BackendType: "extension",
+		BackendData: data,
+	}
+
+	lease, err := be.sm.AcquireLease(ctx, &attrs)
+	switch err {
+	case nil:
+		n.lease = lease
+
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+
+	if len(n.postStartupCommand) > 0 {
+		cmd_output, err := runCmd([]string{
+			fmt.Sprintf("SUBNET=%s", lease.Subnet),
+			fmt.Sprintf("PUBLIC_IP=%s", attrs.PublicIP)},
+			"", "sh", "-c", n.postStartupCommand)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run command: %s Err: %v Output: %s", n.postStartupCommand, err, cmd_output)
+		} else {
+			log.Infof("Ran command: %s\n Output: %s", n.postStartupCommand, cmd_output)
+		}
+	} else {
+		log.Infof("No post startup command configured - skipping")
+	}
+
+	return n, nil
+}
+
+// Run a cmd, returning a combined stdout and stderr.
+func runCmd(env []string, stdin string, name string, arg ...string) (string, error) {
+	cmd := exec.Command(name, arg...)
+	cmd.Env = env
+
+	stdinpipe, err := cmd.StdinPipe()
+	if err != nil {
+		return "", err
+	}
+
+	io.WriteString(stdinpipe, stdin)
+	io.WriteString(stdinpipe, "\n")
+	stdinpipe.Close()
+
+	output, err := cmd.CombinedOutput()
+
+	return strings.TrimSpace(string(output)), err
+}

backend/extension/extension_network.go

@@ -0,0 +1,136 @@
+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extension
+
+import (
+	"encoding/json"
+	"sync"
+
+	log "github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	"fmt"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/subnet"
+)
+
+type network struct {
+	name                string
+	extIface            *backend.ExternalInterface
+	lease               *subnet.Lease
+	sm                  subnet.Manager
+	preStartupCommand   string
+	postStartupCommand  string
+	subnetAddCommand    string
+	subnetRemoveCommand string
+}
+
+func (n *network) Lease() *subnet.Lease {
+	return n.lease
+}
+
+func (n *network) MTU() int {
+	return n.extIface.Iface.MTU
+}
+
+func (n *network) Run(ctx context.Context) {
+	wg := sync.WaitGroup{}
+
+	log.Info("Watching for new subnet leases")
+	evts := make(chan []subnet.Event)
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.lease, evts)
+		wg.Done()
+	}()
+
+	defer wg.Wait()
+
+	for {
+		select {
+		case evtBatch := <-evts:
+			n.handleSubnetEvents(evtBatch)
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
+
+			if evt.Lease.Attrs.BackendType != "extension" {
+				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if len(n.subnetAddCommand) > 0 {
+				var dat interface{}
+				if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &dat); err != nil {
+					log.Errorf("failed to unmarshal BackendData: %v", err)
+				} else {
+					backendData := dat.(string)
+					cmd_output, err := runCmd([]string{
+						fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
+						fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
+						backendData,
+						"sh", "-c", n.subnetAddCommand)
+
+					if err != nil {
+						log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetAddCommand, err, cmd_output)
+					} else {
+						log.Infof("Ran command: %s\n Output: %s", n.subnetAddCommand, cmd_output)
+					}
+				}
+			}
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "extension" {
+				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if len(n.subnetRemoveCommand) > 0 {
+				var dat interface{}
+				if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &dat); err != nil {
+					log.Errorf("failed to unmarshal BackendData: %v", err)
+				} else {
+					backendData := dat.(string)
+					cmd_output, err := runCmd([]string{
+						fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
+						fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
+						backendData,
+						"sh", "-c", n.subnetRemoveCommand)
+
+					if err != nil {
+						log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetRemoveCommand, err, cmd_output)
+					} else {
+						log.Infof("Ran command: %s\n Output: %s", n.subnetRemoveCommand, cmd_output)
+					}
+				}
+			}
+
+		default:
+			log.Error("Internal error: unknown event type: ", int(evt.Type))
+		}
+	}
+}

dist/extension-hostgw

@@ -0,0 +1,8 @@
+{
+  "Network": "10.50.0.0/16",
+  "Backend": {
+    "Type": "extension",
+    "SubnetAddCommand": "ip route add $SUBNET via $PUBLIC_IP",
+    "SubnetRemoveCommand": "ip route del $SUBNET via $PUBLIC_IP"
+  }
+}

dist/extension-vxlan

@@ -0,0 +1,10 @@
+{
+  "Network": "10.50.0.0/16",
+  "Backend": {
+    "Type": "extension",
+    "PreStartupCommand": "export VNI=1; export IF_NAME=flannel-vxlan; ip link del $IF_NAME 2>/dev/null; ip link add $IF_NAME type vxlan id $VNI dstport 8472 nolearning && ip link set mtu 1450 dev $IF_NAME && cat /sys/class/net/$IF_NAME/address",
+    "PostStartupCommand": "export IF_NAME=flannel-vxlan; export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; ip addr add $SUBNET_IP/32 dev $IF_NAME && ip link set $IF_NAME up",
+    "ShutdownCommand": "export IF_NAME=flannel-vxlan; ip link del $IF_NAME",
+    "SubnetAddCommand": "export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; export IF_NAME=flannel-vxlan; read VTEP; ip route add $SUBNET nexthop via $SUBNET_IP dev $IF_NAME onlink && arp -s $SUBNET_IP $VTEP dev $IF_NAME && bridge fdb add $VTEP dev $IF_NAME self dst $PUBLIC_IP"
+  }
+}

dist/functional-test.sh

@@ -39,7 +39,12 @@ docker_version_check() {
 run_test() {
 	backend=$1
 
-	flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" } }"
+    if [ -e "$backend" ]; then
+        echo "Reading custom conf from $backend"
+        flannel_conf=`cat "$backend"`
+    else
+	    flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" } }"
+    fi
 
 	# etcd might take a bit to come up
 	while ! docker run --rm -it $ETCD_IMG etcdctl --endpoints=$etcd_endpt set /coreos.com/network/config "$flannel_conf"