Merge pull request #5 from spidernet-io/route

fix: failed to add route due to wrong scope

Makefile

@@ -12,6 +12,7 @@ build:
 	@mkdir -p ./.tmp/bin ; \
 	for plugin in `ls ./plugins/` ; do   \
 		echo "\033[35m ==> building $${plugin} to $(ROOT_DIR)/.tmp/bin/${plugin}  \033[0m" ; \
+		echo "\033[35m ==> $(GO_BUILD_FLAGS) $(GO_BUILD) $(GO_BUILD_LDFLGAS) -o ./.tmp/bin/$${plugin} ./plugins/$${plugin} \033[0m";  \
 		$(GO_BUILD_FLAGS) $(GO_BUILD) $(GO_BUILD_LDFLGAS) -o ./.tmp/bin/$${plugin} ./plugins/$${plugin} ;  \
 	done
 

pkg/networking/networking.go

@@ -20,7 +20,7 @@ import (
 var DefaultInterfacesToExclude = []string{
 	"docker.*", "cbr.*", "dummy.*",
 	"virbr.*", "lxcbr.*", "veth.*", "lo",
-	"cali.*", "tunl.*", "flannel.*", "kube-ipvs.*", "cni.*",
+	"cali.*", "tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "vx-submariner",
 }
 
 func GetIPFamily(prevResult cnitypes.Result) (int, error) {

pkg/networking/route.go

@@ -1,14 +1,15 @@
 package networking
 
 import (
+	"fmt"
 	"github.com/vishvananda/netlink"
 	"go.uber.org/zap"
 	"net"
 	"os"
 )
 
-func AddRouteTable(logger *zap.Logger, ruleTable int, iface string, destinations []string) error {
-	link, err := netlink.LinkByName(iface)
+func AddRouteTable(logger *zap.Logger, ruleTable int, scope netlink.Scope, device string, destinations []string, v4Gw, v6Gw net.IP) error {
+	link, err := netlink.LinkByName(device)
 	if err != nil {
 		logger.Error(err.Error())
 		return err
@@ -21,15 +22,44 @@ func AddRouteTable(logger *zap.Logger, ruleTable int, iface string, destinations
 			return err
 		}
 
-		if err = netlink.RouteAdd(&netlink.Route{
+		route := &netlink.Route{
 			LinkIndex: link.Attrs().Index,
-			Scope:     netlink.SCOPE_LINK,
+			Scope:     scope,
 			Dst:       ipNet,
 			Table:     ruleTable,
-		}); err != nil && !os.IsExist(err) {
-			logger.Error("failed to add route", zap.String("interface", iface), zap.String("dst", ipNet.String()), zap.Error(err))
+		}
+
+		if ipNet.IP.To4() != nil && v4Gw != nil {
+			route.Gw = v4Gw
+		}
+
+		if ipNet.IP.To4() == nil && v6Gw != nil {
+			route.Gw = v6Gw
+		}
+
+		if err = netlink.RouteAdd(route); err != nil && !os.IsExist(err) {
+			logger.Error("failed to RouteAdd", zap.String("route", route.String()), zap.Error(err))
 			return err
 		}
 	}
 	return nil
 }
+
+func GetGatewayIP(addrs []netlink.Addr) (v4Gw, v6Gw net.IP, err error) {
+	for _, addr := range addrs {
+		routes, err := netlink.RouteGet(addr.IP)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to RouteGet Pod IP(%s): %v", addr.IP.String(), err)
+		}
+
+		if len(routes) > 0 {
+			if addr.IP.To4() != nil && v4Gw == nil {
+				v4Gw = routes[0].Src
+			}
+			if addr.IP.To4() == nil && v6Gw == nil {
+				v6Gw = routes[0].Src
+			}
+		}
+	}
+	return
+}

plugins/veth/veth.go

@@ -111,9 +111,9 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 
 	if !isfirstInterface {
-		logger.Info("Calling veth plugin not for the first time", zap.Any("config", conf))
+		logger.Info("Calling veth plugin not for the first time", zap.Any("config", conf), zap.String("netns", netns.Path()))
 	} else {
-		logger.Info("Calling veth plugin for first time", zap.Any("config", conf))
+		logger.Info("Calling veth plugin for first time", zap.Any("config", conf), zap.String("netns", netns.Path()))
 	}
 
 	var hostVethPairName string
@@ -270,11 +270,17 @@ func setupNeighborhood(logger *zap.Logger, netns ns.NetNS, hostVethPairName stri
 // setupRoutes setup routes for pod and host
 // equivalent to: `ip route add $route`
 func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPairName string, ipAddressOnNode, preInterfaceIPAddress []netlink.Addr, conf *ptypes.Veth) error {
-	err := netns.Do(func(_ ns.NetNS) error {
+	v4Gw, v6Gw, err := networking.GetGatewayIP(preInterfaceIPAddress)
+	if err != nil {
+		logger.Error("failed to GetGatewayIP", zap.Error(err))
+		return err
+	}
+
+	err = netns.Do(func(_ ns.NetNS) error {
 		var err error
 		// traffic sent to the node is forwarded via veth0
 		// eq:  "ip r add <ipAddressOnNode> dev veth0 table <ruleTable> "
-		if err = networking.AddRouteTable(logger, ruleTable, defaultConVeth, networking.AddrsToString(ipAddressOnNode)); err != nil {
+		if err = networking.AddRouteTable(logger, ruleTable, netlink.SCOPE_LINK, defaultConVeth, networking.AddrsToString(ipAddressOnNode), nil, nil); err != nil {
 			logger.Error("failed to AddRouteTable for ipAddressOnNode", zap.Error(err))
 			return fmt.Errorf("failed to AddRouteTable for ipAddressOnNode: %v", err)
 		}
@@ -283,7 +289,7 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair
 		// eq: ip route add <cluster/service cidr> dev veth0
 		localCIDRs := append(conf.ClusterCIDR, conf.ServiceCIDR...)
 		localCIDRs = append(localCIDRs, conf.AdditionalCIDR...)
-		if err := networking.AddRouteTable(logger, ruleTable, defaultConVeth, localCIDRs); err != nil {
+		if err = networking.AddRouteTable(logger, ruleTable, netlink.SCOPE_UNIVERSE, defaultConVeth, localCIDRs, v4Gw, v6Gw); err != nil {
 			logger.Error("failed to AddRouteTable for localCIDRs", zap.Error(err))
 			return fmt.Errorf("failed to AddRouteTable for localCIDRs: %v", err)
 		}
@@ -292,7 +298,7 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair
 		// make sure that all traffic to second NIC to lookup table <<ruleTable>>
 		// eq: ip rule add to <preInterfaceIPAddress> lookup table <ruleTable>
 		if ruleTable != unix.RT_TABLE_MAIN {
-			if err := networking.AddToRuleTable(preInterfaceIPAddress, ruleTable); err != nil {
+			if err = networking.AddToRuleTable(preInterfaceIPAddress, ruleTable); err != nil {
 				logger.Error("failed to AddToRuleTable", zap.Error(err))
 				return fmt.Errorf("failed to AddToRuleTable: %v", err)
 			}
@@ -307,7 +313,8 @@ func setupRoutes(logger *zap.Logger, netns ns.NetNS, ruleTable int, hostVethPair
 
 	// set routes for host
 	// equivalent: ip add  <chainedIPs> dev veth-peer on host
-	if err = networking.AddRouteTable(logger, unix.RT_TABLE_MAIN, hostVethPairName, networking.AddrsToString(preInterfaceIPAddress)); err != nil {
+	if err = networking.AddRouteTable(logger, unix.RT_TABLE_MAIN, netlink.SCOPE_UNIVERSE, hostVethPairName, networking.AddrsToString(preInterfaceIPAddress),
+		nil, nil); err != nil {
 		logger.Error("failed to AddRouteTable for preInterfaceIPAddress", zap.Error(err))
 		return fmt.Errorf("failed to AddRouteTable for preInterfaceIPAddress: %v", err)
 	}