fix swiftv2 route issues

cns/middlewares/k8sSwiftV2.go

@@ -226,3 +226,50 @@ func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantP
 
 	return types.Success, ""
 }
+
+func (k *K8sSWIFTv2Middleware) AddRoutes(cidrs []string, gatewayIP string) []cns.Route {
+	routes := make([]cns.Route, len(cidrs))
+	for i, cidr := range cidrs {
+		routes[i] = cns.Route{
+			IPAddress:        cidr,
+			GatewayIPAddress: gatewayIP,
+		}
+	}
+	return routes
+}
+
+// CNS gets node and service CIDRs from configuration env and parse them to get the v4 and v6 IPs
+func (k *K8sSWIFTv2Middleware) GetCidrs() (v4IPs, v6IPs []string, err error) {
+	v4Cidrs := []string{}
+	v6Cidrs := []string{}
+
+	// Get and parse infraVNETCIDRs from env
+	infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
+	}
+	infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs")
+	}
+
+	// Add infravnet CIDRs to v4 and v6 IPs
+	v4Cidrs = append(v4Cidrs, infraVNETCIDRsv4...)
+	v6Cidrs = append(v6Cidrs, infraVNETCIDRsv6...)
+
+	// Get and parse serviceCIDRs from env
+	serviceCIDRs, err := configuration.ServiceCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env")
+	}
+	serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs")
+	}
+
+	// Add service CIDRs to v4 and v6 IPs
+	v4Cidrs = append(v4Cidrs, serviceCIDRsV4...)
+	v6Cidrs = append(v6Cidrs, serviceCIDRsV6...)
+
+	return v4Cidrs, v6Cidrs, nil
+}

cns/middlewares/k8sSwiftV2_linux.go

@@ -32,50 +32,12 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 		routes = append(routes, virtualGWRoute, route)
 
 	case cns.InfraNIC:
-		// Get and parse infraVNETCIDRs from env
-		infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
+		// get service and infravnet routes
+		infraRoutes, err := k.getInfraRoutes(podIPInfo)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
-		}
-		infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
-		}
-
-		// Get and parse podCIDRs from env
-		podCIDRs, err := configuration.PodCIDRs()
-		if err != nil {
-			return errors.Wrapf(err, "failed to get podCIDRs from env")
-		}
-		podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse podCIDRs")
-		}
-
-		// Get and parse serviceCIDRs from env
-		serviceCIDRs, err := configuration.ServiceCIDRs()
-		if err != nil {
-			return errors.Wrapf(err, "failed to get serviceCIDRs from env")
-		}
-		serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse serviceCIDRs")
-		}
-
-		ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
-		}
-
-		if ip.Is4() {
-			routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...)
-			routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...)
-			routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...)
-		} else {
-			routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...)
-			routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...)
-			routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...)
+			return errors.Wrap(err, "failed to get infra routes for infraNIC interface")
 		}
+		routes = infraRoutes
 		podIPInfo.SkipDefaultRoutes = true
 
 	case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check
@@ -88,15 +50,57 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 	return nil
 }
 
-func addRoutes(cidrs []string, gatewayIP string) []cns.Route {
-	routes := make([]cns.Route, len(cidrs))
-	for i, cidr := range cidrs {
-		routes[i] = cns.Route{
-			IPAddress:        cidr,
-			GatewayIPAddress: gatewayIP,
-		}
+// CNS gets pod CIDRs from configuration env and parse them to get the v4 and v6 IPs
+// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default, so windows swiftv2 does not require pod cidr
+func (k *K8sSWIFTv2Middleware) GetPodCidrs() (v4IPs, v6IPs []string, err error) {
+	v4PodCidrs := []string{}
+	v6PodCidrs := []string{}
+
+	// Get and parse podCIDRs from env
+	podCIDRs, err := configuration.PodCIDRs()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env")
+	}
+	podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs")
+	}
+
+	v4PodCidrs = append(v4PodCidrs, podCIDRsV4...)
+	v6PodCidrs = append(v6PodCidrs, podCIDRv6...)
+
+	return v4PodCidrs, v6PodCidrs, nil
+}
+
+func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
+	var routes []cns.Route
+
+	ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
 	}
-	return routes
+
+	v4IPs, v6IPs, err := k.GetCidrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get node and service CIDRs")
+	}
+
+	v4PodIPs, v6PodIPs, err := k.GetPodCidrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get pod CIDRs")
+	}
+
+	v4IPs = append(v4IPs, v4PodIPs...)
+	v6IPs = append(v6IPs, v6PodIPs...)
+
+	// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway
+	if ip.Is4() {
+		routes = append(routes, k.AddRoutes(v4IPs, overlayGatewayv4)...)
+	} else {
+		routes = append(routes, k.AddRoutes(v6IPs, overlayGatewayV6)...)
+	}
+
+	return routes, nil
 }
 
 // assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient

cns/middlewares/k8sSwiftV2_linux_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"reflect"
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/configuration"
@@ -345,7 +346,7 @@ func TestSetRoutesSuccess(t *testing.T) {
 
 	}
 	for i := range podIPInfo {
-		assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
+		reflect.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
 	}
 }
 
@@ -378,9 +379,10 @@ func TestSetRoutesFailure(t *testing.T) {
 }
 
 func TestAddRoutes(t *testing.T) {
+	middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
 	cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"}
 	gatewayIP := "192.168.1.1"
-	routes := addRoutes(cidrs, gatewayIP)
+	routes := middleware.addRoutes(cidrs, gatewayIP)
 	expectedRoutes := []cns.Route{
 		{
 			IPAddress:        "10.0.0.0/24",

cns/middlewares/k8sSwiftV2_windows.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/netip"
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
@@ -18,6 +19,10 @@ var defaultDenyEgressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionT
 
 var defaultDenyIngressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionTypeIn)
 
+const (
+	defaultGateway = "0.0.0.0"
+)
+
 // for AKS L1VH, do not set default route on infraNIC to avoid customer pod reaching all infra vnet services
 // default route is set for secondary interface NIC(i.e,delegatedNIC)
 func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
@@ -27,10 +32,16 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 		// TODO: Remove this once HNS fix is ready
 		route := cns.Route{
 			IPAddress:        "0.0.0.0/0",
-			GatewayIPAddress: "0.0.0.0",
+			GatewayIPAddress: defaultGateway,
 		}
 		podIPInfo.Routes = append(podIPInfo.Routes, route)
 
+		// set routes(infravnet and service cidrs) for infraNIC interface
+		infraRoutes, err := k.getInfraRoutes(podIPInfo)
+		if err != nil {
+			return errors.Wrap(err, "failed to set routes for infraNIC interface")
+		}
+		podIPInfo.Routes = append(podIPInfo.Routes, infraRoutes...)
 		podIPInfo.SkipDefaultRoutes = true
 	}
 	return nil
@@ -208,3 +219,27 @@ func GetDefaultDenyBool(mtpnc v1alpha1.MultitenantPodNetworkConfig) bool {
 	// returns the value of DefaultDenyACL from mtpnc
 	return mtpnc.Status.DefaultDenyACL
 }
+
+func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
+	var routes []cns.Route
+
+	ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
+	}
+
+	// swiftv2 windows does not support ipv6
+	v4IPs, _, err := k.GetCidrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get CIDRs")
+	}
+
+	if ip.Is4() {
+		// add routes to podIPInfo for the given CIDRs and gateway IP
+		// always use default gateway IP for containerd to configure routes;
+		// containerd will set route with default gateway ip like 10.0.0.0/16 via 0.0.0.0 dev eth0
+		routes = append(routes, k.AddRoutes(v4IPs, defaultGateway)...)
+	}
+
+	return routes, nil
+}

cns/middlewares/k8sSwiftV2_windows_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/configuration"
 	"github.com/Azure/azure-container-networking/cns/middlewares/mock"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
 	"github.com/Azure/azure-container-networking/network/policy"
@@ -17,11 +18,13 @@ import (
 
 func TestSetRoutesSuccess(t *testing.T) {
 	middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
+	t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16")
+	t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.10/16")
 
 	podIPInfo := []cns.PodIpInfo{
 		{
 			PodIPConfig: cns.IPSubnet{
-				IPAddress:    "10.0.1.10",
+				IPAddress:    "10.0.1.100",
 				PrefixLength: 32,
 			},
 			NICType: cns.InfraNIC,
@@ -35,6 +38,30 @@ func TestSetRoutesSuccess(t *testing.T) {
 			MacAddress: "12:34:56:78:9a:bc",
 		},
 	}
+
+	desiredPodIPInfo := []cns.PodIpInfo{
+		{
+			PodIPConfig: cns.IPSubnet{
+				IPAddress:    "10.0.1.100",
+				PrefixLength: 32,
+			},
+			NICType: cns.InfraNIC,
+			Routes: []cns.Route{
+				{
+					IPAddress:        "10.0.0.0/16",
+					GatewayIPAddress: "0.0.0.0",
+				},
+				{
+					IPAddress:        "10.240.0.10/16",
+					GatewayIPAddress: "0.0.0.0",
+				},
+				{
+					IPAddress:        "0.0.0.0/0",
+					GatewayIPAddress: "0.0.0.0",
+				},
+			},
+		},
+	}
 	for i := range podIPInfo {
 		ipInfo := &podIPInfo[i]
 		err := middleware.setRoutes(ipInfo)
@@ -45,6 +72,9 @@ func TestSetRoutesSuccess(t *testing.T) {
 			assert.Equal(t, ipInfo.SkipDefaultRoutes, false)
 		}
 	}
+
+	// check if the routes are set as expected
+	reflect.DeepEqual(podIPInfo[0].Routes, desiredPodIPInfo[0].Routes)
 }
 
 func TestAssignSubnetPrefixSuccess(t *testing.T) {