Skip to content

Commit

Permalink
add L1VH IB support on CNI (#2762)
Browse files Browse the repository at this point in the history 
* add L1VH IB support on CNI

* fix IB issues

* fix UT errors

* fix linter issues

* add win 2025 support for cni image build

* add and comments

* fix a logic bug

* disable endpoint creation and deletion if it's IB NIC

* fix a linter issue

* add UTs

* add UTs for powershell

* enhance Test_getInterfaceInfoKey test case

* remove windows 2025 build from pipeline

* fix some issues

* add an UT to test pnpID

* fix an issue

* fix an ut

* add double quotes

* unblock a brunch of issues

* remove unnecessary codes

* upgradelatest upstream cnii build

* fix a log

* add windows build on pipeline temporarily

* remove backendNIC check for findMasterInterface

* add ut to confirm IB does not create endpoint

* fix linter issue that use %q

* format network.go

* add more uts to cover powershell commands

* remove windows2025 pipeline build

* enhance logs

* fix cniResult format

* add getPnpidstate func

* fix the issue for infraNIC routes

* fix the issue for infraNIC routes

* fix gateway ip address

* add get-pnpdevice UT

* add accelnetNIC support for L1VH

* enhance logic for accelnet nic netowrk flag

* enhance network windows uts

* fix bitmask operator

* use another PR for accelnet PR

* gofumpt files

* fix comments for functional codes

* add uts

* add more uts

* fix uts

* fix functional codes comments

* Update cni/network/network.go

Co-authored-by: tamilmani1989 <tamanoha@microsoft.com>
Signed-off-by: Paul Yu <129891899+paulyufan2@users.noreply.github.com>

* fix latest comments

* fix an UT

* fix invoker_cns_test.go

* fix ut bugs

* fix ut with SkipDefaultRoutes

* add combination ut

* add combination ut

* add ncGateway address to ut

* fix an ut bug

* fix ut bug

* add unhappy test cases

* add endpoint add and deletion cases

* push mock network creation hns api test cases

* remove network creation hns call

* add uts to mock hns network and endpoint calls

* fix ut linter issues

* add infraNIC only invoker test case

* add unhappy path test case

* remove infraNIC only case

* remove unhappy test case

* re-archetect cni ib codes and test

* remove unnecessary logs

* save endpoint state

* save endpoint object for IB

* fix linter issue

* fix a brunch of linter issues

* fix linter issues

* fix linter issue

* fix ut for returned error msg

* temporary add manifest build for CNS/CNI to pipeline

* feedback fix

* fix linter issue

* add ut to get networkName and networkID

* remove Ankit's PR to build cns image

* revert Ankit's changes back

* remove win2025 build from pipeline

* log error for invalid mac address

* revert convertInterfaceInfoToCniResult impl

* fix feedback

* add crd changes to test

* add win2025 yaml to build image

* pass containerID to cns

* revert changes back for review

* revert changes back for review

* gofumpt endpoint.go

* remove comment

* add latest comments

* Update network/endpoint_windows.go

Co-authored-by: tamilmani1989 <tamanoha@microsoft.com>
Signed-off-by: Paul Yu <129891899+paulyufan2@users.noreply.github.com>

* fix a linter issue

* add error check

* add error check

* gofumpt endpoint windows test file

---------

Signed-off-by: Paul Yu <129891899+paulyufan2@users.noreply.github.com>
Co-authored-by: tamilmani1989 <tamanoha@microsoft.com>
  • Loading branch information
@paulyufan2 @tamilmani1989
paulyufan2 and tamilmani1989 authored Jul 11, 2024
1 parent 67b5827 commit 03e0447
Show file tree
Hide file tree
Showing 21 changed files with 886 additions and 75 deletions.
2 changes: 1 addition & 1 deletion cni/azure-windows-swift.conflist
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"cniVersion": "0.3.0",
"cniVersion": "1.0.0",
"name": "azure",
"adapterName" : "",
"plugins": [
Expand Down
2 changes: 1 addition & 1 deletion cni/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ const (
)

// Supported CNI versions.
var supportedVersions = []string{"0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0"}
var supportedVersions = []string{"0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0", "1.0.0"}

// CNI contract.
type PluginApi interface {
Expand Down
37 changes: 35 additions & 2 deletions cni/network/invoker_cns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type IPResultInfo struct {
macAddress string
skipDefaultRoutes bool
routes []cns.Route
pnpID string
}

func (i IPResultInfo) MarshalLogObject(encoder zapcore.ObjectEncoder) error {
Expand Down Expand Up @@ -143,6 +144,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro

addResult := IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)}
numInterfacesWithDefaultRoutes := 0

for i := 0; i < len(response.PodIPInfo); i++ {
info := IPResultInfo{
podIPAddress: response.PodIPInfo[i].PodIPConfig.IPAddress,
Expand All @@ -156,6 +158,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
macAddress: response.PodIPInfo[i].MacAddress,
skipDefaultRoutes: response.PodIPInfo[i].SkipDefaultRoutes,
routes: response.PodIPInfo[i].Routes,
pnpID: response.PodIPInfo[i].PnPID,
}

logger.Info("Received info for pod",
Expand All @@ -167,7 +170,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
key := invoker.getInterfaceInfoKey(info.nicType, info.macAddress)
switch info.nicType {
case cns.DelegatedVMNIC:
// only handling single v4 PodIPInfo for Frontend NICs at the moment, will have to update once v6 gets added
// only handling single v4 PodIPInfo for DelegatedVMNIC at the moment, will have to update once v6 gets added
if !info.skipDefaultRoutes {
numInterfacesWithDefaultRoutes++
}
Expand All @@ -180,6 +183,12 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig, key); err != nil {
return IPAMAddResult{}, err
}
case cns.BackendNIC:
// TODO: check whether setting default route on IB interface
// handle ipv4 PodIPInfo for BackendNIC
if err := addBackendNICToResult(&info, &addResult, key); err != nil {
return IPAMAddResult{}, err
}
case cns.InfraNIC, "":
// if we change from legacy cns, the nicType will be empty, so we assume it is infra nic
info.nicType = cns.InfraNIC
Expand Down Expand Up @@ -464,6 +473,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p

macAddress, err := net.ParseMAC(info.macAddress)
if err != nil {
logger.Error("Invalid mac address", zap.Error(err))
return errors.Wrap(err, "Invalid mac address")
}

Expand Down Expand Up @@ -491,8 +501,31 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p
return nil
}

func addBackendNICToResult(info *IPResultInfo, addResult *IPAMAddResult, key string) error {
macAddress, err := net.ParseMAC(info.macAddress)
if err != nil {
logger.Error("Invalid mac address", zap.Error(err))
return errors.Wrap(err, "Invalid mac address")
}

// return error if pnp id is missing in cns goalstate
if info.pnpID == "" {
logger.Error("pnp id is not received from cns")
return errors.Wrap(err, "pnp id is not received from cns")
}

addResult.interfaceInfo[key] = network.InterfaceInfo{
NICType: info.nicType,
MacAddress: macAddress,
SkipDefaultRoutes: info.skipDefaultRoutes,
PnPID: info.pnpID,
}

return nil
}

func (invoker *CNSIPAMInvoker) getInterfaceInfoKey(nicType cns.NICType, macAddress string) string {
if nicType == cns.DelegatedVMNIC {
if nicType == cns.DelegatedVMNIC || nicType == cns.BackendNIC {
return macAddress
}
return string(nicType)
Expand Down
217 changes: 210 additions & 7 deletions cni/network/invoker_cns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) {
wantErr: false,
},
{
name: "Test happy CNI add with multitenant result",
name: "Test happy CNI add with InfraNIC + DelegatedNIC interfaces",
fields: fields{
podName: testPodInfo.PodName,
podNamespace: testPodInfo.PodNamespace,
Expand Down Expand Up @@ -496,7 +496,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) {
if ifInfo.NICType == cns.DelegatedVMNIC {
fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ifInfo)
if len(tt.wantSecondaryInterfacesInfo.IPConfigs) > 0 {
require.EqualValues(tt.wantSecondaryInterfacesInfo, ifInfo, "incorrect multitenant response")
require.EqualValues(tt.wantSecondaryInterfacesInfo, ifInfo, "incorrect response for delegatedNIC")
}
}
if ifInfo.NICType == cns.InfraNIC {
Expand Down Expand Up @@ -1444,7 +1444,8 @@ func Test_getInterfaceInfoKey(t *testing.T) {
require.Equal(string(cns.InfraNIC), inv.getInterfaceInfoKey(cns.InfraNIC, dummyMAC))
require.Equal(dummyMAC, inv.getInterfaceInfoKey(cns.DelegatedVMNIC, dummyMAC))
require.Equal("", inv.getInterfaceInfoKey(cns.DelegatedVMNIC, ""))
require.Equal(string(cns.NodeNetworkInterfaceBackendNIC), inv.getInterfaceInfoKey(cns.NodeNetworkInterfaceBackendNIC, dummyMAC))
require.Equal(dummyMAC, inv.getInterfaceInfoKey(cns.BackendNIC, dummyMAC))
require.Equal("", inv.getInterfaceInfoKey(cns.BackendNIC, ""))
}

func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
Expand All @@ -1453,6 +1454,11 @@ func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
macAddress := "12:34:56:78:9a:bc"
parsedMacAddress, _ := net.ParseMAC(macAddress)

ibMacAddress := "bc:9a:78:56:34:12"
ibParsedMacAddress, _ := net.ParseMAC(ibMacAddress)

pnpID := "PCI\\VEN_15B3&DEV_101C&SUBSYS_000715B3&REV_00\\5&8c5acce&0&0"

type fields struct {
podName string
podNamespace string
Expand All @@ -1470,11 +1476,12 @@ func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
name string
fields fields
args args
wantDefaultResult network.InterfaceInfo
wantSecondaryInterfacesInfo map[string]network.InterfaceInfo
wantErr bool
}{
{
name: "Test happy CNI add with swiftv2 multitenant result",
name: "Test happy CNI add delegatedVMNIC type",
fields: fields{
podName: testPodInfo.PodName,
podNamespace: testPodInfo.PodNamespace,
Expand Down Expand Up @@ -1535,6 +1542,190 @@ func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
},
wantErr: false,
},
{
name: "Test happy CNI add with DelegatedNIC + BackendNIC interfaces",
fields: fields{
podName: testPodInfo.PodName,
podNamespace: testPodInfo.PodNamespace,
cnsClient: &MockCNSClient{
require: require,
requestIPs: requestIPsHandler{
ipconfigArgument: cns.IPConfigsRequest{
PodInterfaceID: "testcont-testifname1",
InfraContainerID: "testcontainerid1",
OrchestratorContext: marshallPodInfo(testPodInfo),
},
result: &cns.IPConfigsResponse{
PodIPInfo: []cns.PodIpInfo{
{
PodIPConfig: cns.IPSubnet{
IPAddress: "10.1.1.10",
PrefixLength: 24,
},
HostPrimaryIPInfo: cns.HostIPInfo{
Gateway: "10.0.0.1",
PrimaryIP: "10.0.0.2",
Subnet: "10.0.0.1/24",
},
NICType: cns.DelegatedVMNIC,
MacAddress: macAddress,
SkipDefaultRoutes: false,
},
{
MacAddress: ibMacAddress,
NICType: cns.BackendNIC,
PnPID: pnpID,
},
},
Response: cns.Response{
ReturnCode: 0,
Message: "",
},
},
err: nil,
},
},
},
args: args{
nwCfg: &cni.NetworkConfig{},
args: &cniSkel.CmdArgs{
ContainerID: "testcontainerid1",
Netns: "testnetns1",
IfName: "testifname1",
},
hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"),
options: map[string]interface{}{},
},
wantSecondaryInterfacesInfo: map[string]network.InterfaceInfo{
macAddress: {
IPConfigs: []*network.IPConfig{
{
Address: *getCIDRNotationForAddress("10.1.1.10/24"),
},
},
Routes: []network.RouteInfo{},
NICType: cns.DelegatedVMNIC,
MacAddress: parsedMacAddress,
},
ibMacAddress: {
NICType: cns.BackendNIC,
MacAddress: ibParsedMacAddress,
PnPID: pnpID,
},
},
wantErr: false,
},
{
name: "Test happy CNI add with InfraNIC + DelegatedNIC + BackendNIC interfaces",
fields: fields{
podName: testPodInfo.PodName,
podNamespace: testPodInfo.PodNamespace,
cnsClient: &MockCNSClient{
require: require,
requestIPs: requestIPsHandler{
ipconfigArgument: cns.IPConfigsRequest{
PodInterfaceID: "testcont-testifname1",
InfraContainerID: "testcontainerid1",
OrchestratorContext: marshallPodInfo(testPodInfo),
},
result: &cns.IPConfigsResponse{
PodIPInfo: []cns.PodIpInfo{
{
PodIPConfig: cns.IPSubnet{
IPAddress: "10.0.1.10",
PrefixLength: 24,
},
NetworkContainerPrimaryIPConfig: cns.IPConfiguration{
IPSubnet: cns.IPSubnet{
IPAddress: "10.0.1.0",
PrefixLength: 24,
},
DNSServers: nil,
GatewayIPAddress: "10.0.0.1",
},
HostPrimaryIPInfo: cns.HostIPInfo{
Gateway: "10.0.0.1",
PrimaryIP: "10.0.0.1",
Subnet: "10.0.0.0/24",
},
NICType: cns.InfraNIC,
SkipDefaultRoutes: true,
},
{
PodIPConfig: cns.IPSubnet{
IPAddress: "20.1.1.10",
PrefixLength: 24,
},
HostPrimaryIPInfo: cns.HostIPInfo{
Gateway: "20.0.0.1",
PrimaryIP: "20.0.0.2",
Subnet: "20.0.0.1/24",
},
NICType: cns.DelegatedVMNIC,
MacAddress: macAddress,
SkipDefaultRoutes: false,
},
{
MacAddress: ibMacAddress,
NICType: cns.BackendNIC,
PnPID: pnpID,
},
},
Response: cns.Response{
ReturnCode: 0,
Message: "",
},
},
err: nil,
},
},
},
args: args{
nwCfg: &cni.NetworkConfig{},
args: &cniSkel.CmdArgs{
ContainerID: "testcontainerid1",
Netns: "testnetns1",
IfName: "testifname1",
},
hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"),
options: map[string]interface{}{},
},
wantDefaultResult: network.InterfaceInfo{
IPConfigs: []*network.IPConfig{
{
Address: *getCIDRNotationForAddress("10.0.1.10/24"),
Gateway: net.ParseIP("10.0.0.1"),
},
},
Routes: []network.RouteInfo{
{
Dst: network.Ipv4DefaultRouteDstPrefix,
Gw: net.ParseIP("10.0.0.1"),
},
},
NICType: cns.InfraNIC,
SkipDefaultRoutes: true,
HostSubnetPrefix: *parseCIDR("10.0.0.0/24"),
},
wantSecondaryInterfacesInfo: map[string]network.InterfaceInfo{
macAddress: {
IPConfigs: []*network.IPConfig{
{
Address: *getCIDRNotationForAddress("20.1.1.10/24"),
},
},
Routes: []network.RouteInfo{},
NICType: cns.DelegatedVMNIC,
MacAddress: parsedMacAddress,
},
ibMacAddress: {
NICType: cns.BackendNIC,
MacAddress: ibParsedMacAddress,
PnPID: pnpID,
},
},
wantErr: false,
},
}
for _, tt := range tests {
tt := tt
Expand All @@ -1551,9 +1742,21 @@ func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
require.NoError(err)
}

fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.interfaceInfo)
if len(tt.wantSecondaryInterfacesInfo[macAddress].IPConfigs) > 0 {
require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.interfaceInfo, "incorrect multitenant response")
for _, ifInfo := range ipamAddResult.interfaceInfo {
if ifInfo.NICType == cns.InfraNIC {
fmt.Printf("want:%+v\nrest:%+v\n", tt.wantDefaultResult, ifInfo)
require.Equalf(tt.wantDefaultResult, ifInfo, "incorrect ipv4 response")
}

if ifInfo.NICType == cns.BackendNIC {
fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.interfaceInfo[ibMacAddress])
require.EqualValues(tt.wantSecondaryInterfacesInfo[ibMacAddress], ipamAddResult.interfaceInfo[ibMacAddress], "incorrect multitenant response for IB")
}

if ifInfo.NICType == cns.DelegatedVMNIC {
fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo[macAddress], ipamAddResult.interfaceInfo[macAddress])
require.EqualValues(tt.wantSecondaryInterfacesInfo[macAddress], ipamAddResult.interfaceInfo[macAddress], "incorrect multitenant response for Delegated")
}
}
})
}
Expand Down
Loading

0 comments on commit 03e0447

Please sign in to comment.