diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a11290c205..ee2372b2e7 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -36,124 +36,33 @@ type K8sSWIFTv2Middleware struct { // Verify interface compliance at compile time var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) -// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request -// and release IP configs handlers. -func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { - return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req) - - if respCode != types.Success { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: respCode, - Message: message, - }, - }, errors.New("failed to validate IP configs request") - } - ipConfigsResp, err := defaultHandler(ctx, req) - // If the pod is not v2, return the response from the handler - if !req.SecondaryInterfacesExist { - return ipConfigsResp, err - } - // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config - defer func() { - // Release the default IP config if there is an error - if err != nil { - _, err = failureHandler(ctx, req) - if err != nil { - logger.Errorf("failed to release default IP config : %v", err) - } - } - }() - if err != nil { - return ipConfigsResp, err - } - SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) - } - ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) - // Set routes for the pod - for i := range ipConfigsResp.PodIPInfo { - ipInfo := &ipConfigsResp.PodIPInfo[i] - // Backend nics doesn't need routes to be set - if ipInfo.NICType != cns.BackendNIC { - err = k.setRoutes(ipInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name()) - } - } - } - return ipConfigsResp, nil - } -} - -// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. -// nolint -func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { - // Retrieve the pod from the cluster - podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - if err != nil { - errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) - return nil, types.UnexpectedError, errBuf.Error() - } - logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) - podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - pod := v1.Pod{} - if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { - errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) - return nil, types.UnexpectedError, errBuf.Error() +func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { + // gets pod info for the specified request + podInfo, pod, respCode, message := k.GetPodInfo(ctx, req) + if respCode != types.Success { + return nil, respCode, message } - // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready - _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] - _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] - if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel { + // validates if pod is swiftv2 + isSwiftv2 := ValidateSwiftv2Pod(pod) - // Check if the MTPNC CRD exists for the pod, if not, return error - mtpnc := v1alpha1.MultitenantPodNetworkConfig{} - mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() - } - // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if !mtpnc.IsReady() { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() - } - // If primary Ip is set in status field, it indicates the presence of secondary interfaces - if mtpnc.Status.PrimaryIP != "" { - req.SecondaryInterfacesExist = true + var mtpnc v1alpha1.MultitenantPodNetworkConfig + // if swiftv2 is enabled, get mtpnc + if isSwiftv2 { + mtpnc, respCode, message = k.getMTPNC(ctx, podInfo) + if respCode != types.Success { + return nil, respCode, message } - interfaceInfos := mtpnc.Status.InterfaceInfos - for _, interfaceInfo := range interfaceInfos { - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { - if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() - } - req.BackendInterfaceExist = true - req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) - } - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { - req.SecondaryInterfacesExist = true - } + // update ipConfigRequest + respCode, message = k.UpdateIPConfigRequest(mtpnc, req) + if respCode != types.Success { + return nil, respCode, message } } logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist) - // retrieve podinfo from orchestrator context + return podInfo, types.Success, "" } @@ -249,3 +158,119 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode { return cns.K8sSWIFTV2 } + +// gets Pod Data +func (k *K8sSWIFTv2Middleware) GetPodInfo(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, k8sPod v1.Pod, respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() + } + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() + } + return podInfo, pod, types.Success, "" +} + +// validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. +func ValidateSwiftv2Pod(pod v1.Pod) bool { + // check the pod labels for Swift V2 + _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] + _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] + return swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel +} + +func (k *K8sSWIFTv2Middleware) getMTPNC(ctx context.Context, podInfo cns.PodInfo) (mtpncResource v1alpha1.MultitenantPodNetworkConfig, respCode types.ResponseCode, message string) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + } + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if !mtpnc.IsReady() { + return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, errMTPNCNotReady.Error() + } + return mtpnc, types.Success, "" +} + +// Updates Ip Config Request +func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantPodNetworkConfig, req *cns.IPConfigsRequest) ( + respCode types.ResponseCode, + message string, +) { + // If primary Ip is set in status field, it indicates the presence of secondary interfaces + if mtpnc.Status.PrimaryIP != "" { + req.SecondaryInterfacesExist = true + } + + interfaceInfos := mtpnc.Status.InterfaceInfos + for _, interfaceInfo := range interfaceInfos { + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { + if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } + req.BackendInterfaceExist = true + req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) + + } + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { + req.SecondaryInterfacesExist = true + } + } + + return types.Success, "" +} + +func (k *K8sSWIFTv2Middleware) AddRoutes(cidrs []string, gatewayIP string) []cns.Route { + routes := make([]cns.Route, len(cidrs)) + for i, cidr := range cidrs { + routes[i] = cns.Route{ + IPAddress: cidr, + GatewayIPAddress: gatewayIP, + } + } + return routes +} + +// Both Linux and Windows CNS gets infravnet and service CIDRs from configuration env +// GetCidrs() returns v4CIDRs(infravnet and service cidrs) as first []string and v6CIDRs(infravnet and service) as second []string +func (k *K8sSWIFTv2Middleware) GetCidrs() ([]string, []string, error) { //nolint + v4Cidrs := []string{} + v6Cidrs := []string{} + + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env") + } + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs") + } + + // Add infravnet CIDRs to v4 and v6 IPs + v4Cidrs = append(v4Cidrs, infraVNETCIDRsv4...) + v6Cidrs = append(v6Cidrs, infraVNETCIDRsv6...) + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs") + } + + // Add service CIDRs to v4 and v6 IPs + v4Cidrs = append(v4Cidrs, serviceCIDRsV4...) + v6Cidrs = append(v6Cidrs, serviceCIDRsV6...) + + return v4Cidrs, v6Cidrs, nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index e9a93de0e2..813882de55 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -1,6 +1,7 @@ package middlewares import ( + "context" "fmt" "net/netip" @@ -8,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/middlewares/utils" + "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" ) @@ -30,50 +32,12 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { routes = append(routes, virtualGWRoute, route) case cns.InfraNIC: - // Get and parse infraVNETCIDRs from env - infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + // Linux CNS middleware sets the infra routes(pod, infravnet and service cidrs) to infraNIC interface for the podIPInfo used in SWIFT V2 Linux scenario + infraRoutes, err := k.getInfraRoutes(podIPInfo) if err != nil { - return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") - } - infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse infraVNETCIDRs") - } - - // Get and parse podCIDRs from env - podCIDRs, err := configuration.PodCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get podCIDRs from env") - } - podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse podCIDRs") - } - - // Get and parse serviceCIDRs from env - serviceCIDRs, err := configuration.ServiceCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get serviceCIDRs from env") - } - serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse serviceCIDRs") - } - - ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) - if err != nil { - return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) - } - - if ip.Is4() { - routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...) - } else { - routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...) - routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...) - routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...) + return errors.Wrap(err, "failed to get infra routes for infraNIC interface") } + routes = infraRoutes podIPInfo.SkipDefaultRoutes = true case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check @@ -86,15 +50,60 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { return nil } -func addRoutes(cidrs []string, gatewayIP string) []cns.Route { - routes := make([]cns.Route, len(cidrs)) - for i, cidr := range cidrs { - routes[i] = cns.Route{ - IPAddress: cidr, - GatewayIPAddress: gatewayIP, - } +// Linux CNS gets pod CIDRs from configuration env +// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default on Windows VM +// Hence the windows swiftv2 scenario does not require pod cidr +// GetPodCidrs() will return v4PodCidrs as first []string and v6PodCidrs as second []string +func (k *K8sSWIFTv2Middleware) GetPodCidrs() ([]string, []string, error) { //nolint + v4PodCidrs := []string{} + v6PodCidrs := []string{} + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs") + } + + v4PodCidrs = append(v4PodCidrs, podCIDRsV4...) + v6PodCidrs = append(v6PodCidrs, podCIDRv6...) + + return v4PodCidrs, v6PodCidrs, nil +} + +// getInfraRoutes() returns the infra routes including infravnet/pod/service cidrs for the podIPInfo used in SWIFT V2 Linux scenario +// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway +func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + v4IPs, v6IPs, err := k.GetCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get node and service CIDRs") } - return routes + + v4PodIPs, v6PodIPs, err := k.GetPodCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get pod CIDRs") + } + + v4IPs = append(v4IPs, v4PodIPs...) + v6IPs = append(v6IPs, v6PodIPs...) + + if ip.Is4() { + routes = append(routes, k.AddRoutes(v4IPs, overlayGatewayv4)...) + } else { + routes = append(routes, k.AddRoutes(v6IPs, overlayGatewayV6)...) + } + + return routes, nil } // assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient @@ -102,4 +111,69 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, return nil } +// add default route is done on setRoutes() for Linux swiftv2 func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + +// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request +// and release IP configs handlers. +func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { + return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + podInfo, respCode, message := k.GetPodInfoForIPConfigsRequest(ctx, &req) + + if respCode != types.Success { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: respCode, + Message: message, + }, + }, errors.New("failed to validate IP configs request") + } + ipConfigsResp, err := defaultHandler(ctx, req) + // If the pod is not v2, return the response from the handler + if !req.SecondaryInterfacesExist { + return ipConfigsResp, err + } + // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config + defer func() { + // Release the default IP config if there is an error + if err != nil { + _, err = failureHandler(ctx, req) + if err != nil { + logger.Errorf("failed to release default IP config : %v", err) + } + } + }() + if err != nil { + return ipConfigsResp, err + } + SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) + } + ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) + // Set routes for the pod + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // Backend nics doesn't need routes to be set + if ipInfo.NICType != cns.BackendNIC { + err = k.setRoutes(ipInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name()) + } + } + } + return ipConfigsResp, nil + } +} diff --git a/cns/middlewares/k8sSwiftV2_linux_test.go b/cns/middlewares/k8sSwiftV2_linux_test.go index 76be6b2149..9c627c6c43 100644 --- a/cns/middlewares/k8sSwiftV2_linux_test.go +++ b/cns/middlewares/k8sSwiftV2_linux_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/google/go-cmp/cmp" "gotest.tools/v3/assert" ) @@ -144,7 +145,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.SecondaryInterfacesExist = false - _, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq) + _, respCode, err := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -158,7 +159,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq2.OrchestratorContext = b happyReq2.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2) + _, respCode, err = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq2) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -172,7 +173,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq3.OrchestratorContext = b happyReq3.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3) + _, respCode, err = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq3) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq3.SecondaryInterfacesExist, false) @@ -188,7 +189,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - _, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _ := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -198,19 +199,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Failed to get MTPNC b, _ = testPod3Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // MTPNC not ready b, _ = testPod4Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) } @@ -342,10 +343,10 @@ func TestSetRoutesSuccess(t *testing.T) { } else { assert.Equal(t, ipInfo.SkipDefaultRoutes, false) } - } + for i := range podIPInfo { - assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) + cmp.Equal(podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) } } @@ -378,9 +379,10 @@ func TestSetRoutesFailure(t *testing.T) { } func TestAddRoutes(t *testing.T) { + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"} gatewayIP := "192.168.1.1" - routes := addRoutes(cidrs, gatewayIP) + routes := middleware.AddRoutes(cidrs, gatewayIP) expectedRoutes := []cns.Route{ { IPAddress: "10.0.0.0/24", diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index 2be2fbd1df..9eb681b375 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -1,12 +1,28 @@ package middlewares import ( + "context" + "encoding/json" + "fmt" + "net/netip" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/middlewares/utils" + "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/network/policy" "github.com/pkg/errors" ) +var defaultDenyEgressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionTypeOut) + +var defaultDenyIngressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionTypeIn) + +const ( + defaultGateway = "0.0.0.0" +) + // for AKS L1VH, do not set default route on infraNIC to avoid customer pod reaching all infra vnet services // default route is set for secondary interface NIC(i.e,delegatedNIC) func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { @@ -16,10 +32,16 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { // TODO: Remove this once HNS fix is ready route := cns.Route{ IPAddress: "0.0.0.0/0", - GatewayIPAddress: "0.0.0.0", + GatewayIPAddress: defaultGateway, } podIPInfo.Routes = append(podIPInfo.Routes, route) + // Windows CNS middleware sets the infra routes(infravnet and service cidrs) to infraNIC interface for the podIPInfo used in SWIFT V2 Windows scenario + infraRoutes, err := k.getInfraRoutes(podIPInfo) + if err != nil { + return errors.Wrap(err, "failed to set routes for infraNIC interface") + } + podIPInfo.Routes = append(podIPInfo.Routes, infraRoutes...) podIPInfo.SkipDefaultRoutes = true } return nil @@ -50,7 +72,7 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.Pod return nil } -// add default route with gateway IP to podIPInfo +// add default route with gateway IP to podIPInfo for delegatedNIC interface func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP string) { route := cns.Route{ IPAddress: "0.0.0.0/0", @@ -58,3 +80,166 @@ func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP st } podIPInfo.Routes = append(podIPInfo.Routes, route) } + +func mustGetEndpointPolicy(direction string) policy.Policy { + endpointPolicy, err := getEndpointPolicy(direction) + if err != nil { + panic(err) + } + return endpointPolicy +} + +// get policy of type endpoint policy given the params +func getEndpointPolicy(direction string) (policy.Policy, error) { + endpointPolicy, err := createEndpointPolicy(direction) + if err != nil { + return policy.Policy{}, fmt.Errorf("error creating endpoint policy: %w", err) + } + + additionalArgs := policy.Policy{ + Type: policy.EndpointPolicy, + Data: endpointPolicy, + } + + return additionalArgs, nil +} + +// create policy given the params +func createEndpointPolicy(direction string) ([]byte, error) { + endpointPolicy := struct { + Type string `json:"Type"` + Action string `json:"Action"` + Direction string `json:"Direction"` + Priority int `json:"Priority"` + }{ + Type: string(policy.ACLPolicy), + Action: cns.ActionTypeBlock, + Direction: direction, + Priority: 10_000, + } + + rawPolicy, err := json.Marshal(endpointPolicy) + if err != nil { + return nil, fmt.Errorf("error marshalling policy to json, err is: %w", err) + } + + return rawPolicy, nil +} + +// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request +// and release IP configs handlers. +func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { + return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + podInfo, respCode, message := k.GetPodInfoForIPConfigsRequest(ctx, &req) + + if respCode != types.Success { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: respCode, + Message: message, + }, + }, errors.New("failed to validate IP configs request") + } + ipConfigsResp, err := defaultHandler(ctx, req) + // If the pod is not v2, return the response from the handler + if !req.SecondaryInterfacesExist { + return ipConfigsResp, err + } + + // Get MTPNC + mtpnc, respCode, message := k.getMTPNC(ctx, podInfo) + if respCode != types.Success { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: respCode, + Message: message, + }, + }, errors.New("failed to validate IP configs request") + } + + // GetDefaultDenyBool takes in mtpnc and returns the value of defaultDenyACLBool from it + defaultDenyACLBool := GetDefaultDenyBool(mtpnc) + + // ipConfigsResp has infra IP configs -> if defaultDenyACLbool is enabled, add the default deny endpoint policies as a property in PodIpInfo + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // there will be no pod connectivity to and from those pods + if defaultDenyACLBool && ipInfo.NICType == cns.InfraNIC { + ipInfo.EndpointPolicies = append(ipInfo.EndpointPolicies, defaultDenyEgressPolicy, defaultDenyIngressPolicy) + break + } + } + + // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config + defer func() { + // Release the default IP config if there is an error + if err != nil { + _, err = failureHandler(ctx, req) + if err != nil { + logger.Errorf("failed to release default IP config : %v", err) + } + } + }() + if err != nil { + return ipConfigsResp, err + } + SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) + } + ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) + // Set routes for the pod + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // Backend nics doesn't need routes to be set + if ipInfo.NICType != cns.BackendNIC { + err = k.setRoutes(ipInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name()) + } + } + } + return ipConfigsResp, nil + } +} + +func GetDefaultDenyBool(mtpnc v1alpha1.MultitenantPodNetworkConfig) bool { + // returns the value of DefaultDenyACL from mtpnc + return mtpnc.Status.DefaultDenyACL +} + +// getInfraRoutes() returns the infra routes including infravnet/ and service cidrs for the podIPInfo used in SWIFT V2 Windows scenario +// Windows uses default route 0.0.0.0 as the gateway IP for containerd to configure; +// For example, containerd would set route like: ip route 10.0.0.0/16 via 0.0.0.0 dev eth0 +func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + // TODO: add ipv6 when supported + v4IPs, _, err := k.GetCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get CIDRs") + } + + if ip.Is4() { + routes = append(routes, k.AddRoutes(v4IPs, defaultGateway)...) + } + + return routes, nil +} diff --git a/cns/middlewares/k8sSwiftV2_windows_test.go b/cns/middlewares/k8sSwiftV2_windows_test.go index dab24685f9..bc5464cbb9 100644 --- a/cns/middlewares/k8sSwiftV2_windows_test.go +++ b/cns/middlewares/k8sSwiftV2_windows_test.go @@ -1,22 +1,31 @@ package middlewares import ( + "encoding/json" + "fmt" "reflect" "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/network/policy" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" "gotest.tools/v3/assert" ) func TestSetRoutesSuccess(t *testing.T) { middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16") + t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.10/16") + t.Setenv(configuration.EnvPodCIDRs, "10.1.0.10/24") // make sure windows swiftv2 does not set pod cidr route podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ - IPAddress: "10.0.1.10", + IPAddress: "10.0.1.100", PrefixLength: 32, }, NICType: cns.InfraNIC, @@ -30,6 +39,30 @@ func TestSetRoutesSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", }, } + + desiredPodIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.100", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "10.0.0.0/16", + GatewayIPAddress: "0.0.0.0", + }, + { + IPAddress: "10.240.0.10/16", + GatewayIPAddress: "0.0.0.0", + }, + { + IPAddress: "0.0.0.0/0", + GatewayIPAddress: "0.0.0.0", + }, + }, + }, + } for i := range podIPInfo { ipInfo := &podIPInfo[i] err := middleware.setRoutes(ipInfo) @@ -40,6 +73,9 @@ func TestSetRoutesSuccess(t *testing.T) { assert.Equal(t, ipInfo.SkipDefaultRoutes, false) } } + + // check if the routes are set as expected + reflect.DeepEqual(podIPInfo[0].Routes, desiredPodIPInfo[0].Routes) } func TestAssignSubnetPrefixSuccess(t *testing.T) { @@ -100,3 +136,86 @@ func TestAddDefaultRoute(t *testing.T) { t.Errorf("got '%+v', expected '%+v'", ipInfo.Routes, expectedRoutes) } } + +func TestAddDefaultDenyACL(t *testing.T) { + const policyType = "ACL" + const action = "Block" + const ingressDir = "In" + const egressDir = "Out" + const priority = 10000 + + valueIn := []byte(fmt.Sprintf(`{ + "Type": "%s", + "Action": "%s", + "Direction": "%s", + "Priority": %d + }`, + policyType, + action, + ingressDir, + priority, + )) + + valueOut := []byte(fmt.Sprintf(`{ + "Type": "%s", + "Action": "%s", + "Direction": "%s", + "Priority": %d + }`, + policyType, + action, + egressDir, + priority, + )) + + expectedDefaultDenyEndpoint := []policy.Policy{ + { + Type: policy.EndpointPolicy, + Data: valueOut, + }, + { + Type: policy.EndpointPolicy, + Data: valueIn, + }, + } + var allEndpoints []policy.Policy + var defaultDenyEgressPolicy, defaultDenyIngressPolicy policy.Policy + var err error + + defaultDenyEgressPolicy = mustGetEndpointPolicy("Out") + defaultDenyIngressPolicy = mustGetEndpointPolicy("In") + + allEndpoints = append(allEndpoints, defaultDenyEgressPolicy, defaultDenyIngressPolicy) + + // Normalize both slices so there is no extra spacing, new lines, etc + normalizedExpected := normalizeKVPairs(t, expectedDefaultDenyEndpoint) + normalizedActual := normalizeKVPairs(t, allEndpoints) + if !cmp.Equal(normalizedExpected, normalizedActual) { + t.Error("received policy differs from expectation: diff", cmp.Diff(normalizedExpected, normalizedActual)) + } + assert.Equal(t, err, nil) +} + +// normalizeKVPairs normalizes the JSON values in the KV pairs by unmarshaling them into a map, then marshaling them back to compact JSON to remove any extra space, new lines, etc +func normalizeKVPairs(t *testing.T, policies []policy.Policy) []policy.Policy { + normalized := make([]policy.Policy, len(policies)) + + for i, kv := range policies { + var unmarshaledValue map[string]interface{} + // Unmarshal the Value into a map + err := json.Unmarshal(kv.Data, &unmarshaledValue) + require.NoError(t, err, "Failed to unmarshal JSON value") + + // Marshal it back to compact JSON + normalizedValue, err := json.Marshal(unmarshaledValue) + require.NoError(t, err, "Failed to re-marshal JSON value") + + // Replace Value with the normalized compact JSON + normalized[i] = policy.Policy{ + Type: policy.EndpointPolicy, + Data: normalizedValue, + } + } + + return normalized +}