Merge pull request #178 from sunya-ch/v1.2.0

allow empty master if devices found on pod
foundation-model-stack · Feb 29, 2024 · 1863bb2 · 1863bb2
2 parents 9e657f5 + 1219c2f
commit 1863bb2
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 37 deletions.
diff --git a/cni/plugins/main/multi-nic/host-device.go b/cni/plugins/main/multi-nic/host-device.go
@@ -23,7 +23,7 @@ type HostDeviceTypeNetConf struct {
 	MainPlugin *HostDeviceNetConf `json:"plugin"`
 }
 
-type HostDeviceRuntimConfig struct {
+type HostDeviceRuntimeConfig struct {
 	DeviceID string `json:"deviceID,omitempty"`
 }
 
@@ -33,9 +33,9 @@ type HostDeviceNetConf struct {
 	Device        string `json:"device"` // Device-Name, something like eth0 or can0 etc.
 	HWAddr        string `json:"hwaddr"` // MAC Address of target network interface
 	DPDKMode      bool
-	KernelPath    string                 `json:"kernelpath"` // Kernelpath of the device
-	PCIAddr       string                 `json:"pciBusID"`   // PCI Address of target network device
-	RuntimeConfig HostDeviceRuntimConfig `json:"runtimeConfig,omitempty"`
+	KernelPath    string                  `json:"kernelpath"` // Kernelpath of the device
+	PCIAddr       string                  `json:"pciBusID"`   // PCI Address of target network device
+	RuntimeConfig HostDeviceRuntimeConfig `json:"runtimeConfig,omitempty"`
 }
 
 // loadHostDeviceConf unmarshal to HostDeviceNetConf and returns list of SR-IOV configs
@@ -62,7 +62,7 @@ func loadHostDeviceConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*cu
 			singleConfig.CNIVersion = n.CNIVersion
 		}
 		singleConfig.Name = fmt.Sprintf("%s-%d", ifName, index)
-		singleConfig.RuntimeConfig = HostDeviceRuntimConfig{
+		singleConfig.RuntimeConfig = HostDeviceRuntimeConfig{
 			DeviceID: deviceID,
 		}
 		confBytes, err := json.Marshal(singleConfig)

diff --git a/daemon/src/iface/pci.go b/daemon/src/iface/pci.go
@@ -114,45 +114,42 @@ func GetPodResourceMap(pod *v1.Pod) (map[string][]string, error) {
 }
 
 // GetDeviceMap returns a map from network address to NIC device
-func GetDeviceMap(resourceMap map[string][]string, resourceName string) map[string]string {
+func GetDeviceMap(deviceIDs []string) map[string]string {
 	deviceMap := make(map[string]string)
-	log.Printf("GetDeviceMap of %s\n", resourceName)
 	nameNetMap := GetNameNetMap()
-	log.Printf("resource map: %v\n", resourceMap)
 	log.Printf("nameNetMap map: %v\n", nameNetMap)
 
-	if deviceIDs, exist := resourceMap[resourceName]; exist {
-		for _, deviceID := range deviceIDs {
-			var masterName string
-			deviceNameInterface := deviceMapCache.GetCache(deviceID)
-			if deviceNameInterface != nil {
-				masterName = deviceNameInterface.(string)
+	for _, deviceID := range deviceIDs {
+		var masterName string
+		deviceNameInterface := deviceMapCache.GetCache(deviceID)
+		if deviceNameInterface != nil {
+			masterName = deviceNameInterface.(string)
+		} else {
+			var err error
+			masterName, err = GetPfName(deviceID)
+			if err != nil {
+				log.Printf("cannot get physical device %s: %v\n", deviceID, err)
 			} else {
-				var err error
-				masterName, err = GetPfName(deviceID)
-				if err != nil {
-					log.Printf("cannot get physical device %s: %v\n", deviceID, err)
-				} else {
-					log.Printf("set deviceMapCache %s=%s\n", deviceID, masterName)
-					deviceMapCache.SetCache(deviceID, masterName)
-				}
+				log.Printf("set deviceMapCache %s=%s\n", deviceID, masterName)
+				deviceMapCache.SetCache(deviceID, masterName)
 			}
-			if netAddress, exist := nameNetMap[masterName]; exist {
-				deviceMap[netAddress] = deviceID
+		}
+		if netAddress, exist := nameNetMap[masterName]; exist {
+			deviceMap[netAddress] = deviceID
+		} else {
+			netAddress, err := getNetAddressFromDevice(masterName)
+			if err != nil {
+				log.Printf("cannot get network address of device %s: %v\n", masterName, err)
 			} else {
-				netAddress, err := getNetAddressFromDevice(masterName)
-				if err != nil {
-					log.Printf("cannot get network address of device %s: %v\n", masterName, err)
-				} else {
-					deviceMap[netAddress] = deviceID
-					// found new device, update interfaces
-					GetInterfaces()
-					nameNetMap = GetNameNetMap()
-					log.Printf("updated nameNetMap map: %v\n", nameNetMap)
-				}
+				deviceMap[netAddress] = deviceID
+				// found new device, update interfaces
+				GetInterfaces()
+				nameNetMap = GetNameNetMap()
+				log.Printf("updated nameNetMap map: %v\n", nameNetMap)
 			}
 		}
 	}
+
 	return deviceMap
 }
 

diff --git a/daemon/src/selector/selector.go b/daemon/src/selector/selector.go
@@ -52,7 +52,16 @@ var GPUIdBusIdMap = GetGPUIDMap()
 var TopologyFilePath = defaultTopologyFilePath
 var NumaAwareSelectorInstance = InitNumaAwareSelector(TopologyFilePath, GPUIdBusIdMap)
 
-func getDefaultResponse(req NICSelectRequest, masterNameMap map[string]string, nameNetMap map[string]string, deviceMap map[string]string, resourceMap map[string][]string) NICSelectResponse {
+func isEmptyDeviceIDs(selectedDeviceIDs []string) bool {
+	for _, deviceID := range selectedDeviceIDs {
+		if deviceID != "" {
+			return false
+		}
+	}
+	return true
+}
+
+func getDefaultResponse(req NICSelectRequest, masterNameMap map[string]string, nameNetMap map[string]string, deviceMap map[string]string, resourceMap map[string][]string, podDeviceIDs []string) NICSelectResponse {
 	selector := DefaultSelector{}
 	selectedMasterNetAddrs := selector.Select(req, masterNameMap, nameNetMap, resourceMap)
 	selectedMasters := []string{}
@@ -63,6 +72,15 @@ func getDefaultResponse(req NICSelectRequest, masterNameMap map[string]string, n
 		selectedDeviceIDs = append(selectedDeviceIDs, deviceID)
 		selectedMasters = append(selectedMasters, master)
 	}
+	// allow empty master name for the devices that has been assigned to the pod
+	if len(podDeviceIDs) > 0 && isEmptyDeviceIDs(selectedDeviceIDs) {
+		selectedMasters = []string{}
+		selectedDeviceIDs = []string{}
+		for _, deviceID := range podDeviceIDs {
+			selectedDeviceIDs = append(selectedDeviceIDs, deviceID)
+			selectedMasters = append(selectedMasters, "")
+		}
+	}
 
 	return NICSelectResponse{
 		DeviceIDs: selectedDeviceIDs,
@@ -73,6 +91,7 @@ func getDefaultResponse(req NICSelectRequest, masterNameMap map[string]string, n
 func Select(req NICSelectRequest) NICSelectResponse {
 	deviceMap := make(map[string]string)
 	resourceMap := make(map[string][]string)
+	podDeviceIDs := []string{}
 
 	pod, err := K8sClientset.CoreV1().Pods(req.PodNamespace).Get(context.TODO(), req.PodName, metav1.GetOptions{})
 	if err == nil {
@@ -81,7 +100,12 @@ func Select(req NICSelectRequest) NICSelectResponse {
 			log.Printf("resourceMap of %s: %v\n", pod.UID, resourceMap)
 			resourceName := NetAttachDefHandler.GetResourceName(req.NetAttachDefName, req.PodNamespace)
 			if resourceName != "" {
-				deviceMap = iface.GetDeviceMap(resourceMap, resourceName)
+				log.Printf("resource map: %v\n", resourceMap)
+				if deviceIDs, exist := resourceMap[resourceName]; exist {
+					log.Printf("GetDeviceMap of %s\n", resourceName)
+					podDeviceIDs = deviceIDs
+					deviceMap = iface.GetDeviceMap(podDeviceIDs)
+				}
 				log.Printf("deviceMap: %v\n", deviceMap)
 			}
 		} else {
@@ -95,7 +119,7 @@ func Select(req NICSelectRequest) NICSelectResponse {
 	nameNetMap := iface.GetNameNetMap()
 	netSpec, err := MultinicnetHandler.Get(req.NetAttachDefName, req.PodNamespace)
 	if err != nil {
-		return getDefaultResponse(req, masterNameMap, nameNetMap, deviceMap, resourceMap)
+		return getDefaultResponse(req, masterNameMap, nameNetMap, deviceMap, resourceMap, podDeviceIDs)
 	}
 	policy := netSpec.Policy
 
@@ -138,6 +162,15 @@ func Select(req NICSelectRequest) NICSelectResponse {
 			selectedMasters = append(selectedMasters, master)
 		}
 	}
+	// allow empty master name for the devices that has been assigned to the pod
+	if len(podDeviceIDs) > 0 && isEmptyDeviceIDs(selectedDeviceIDs) {
+		selectedMasters = []string{}
+		selectedDeviceIDs = []string{}
+		for _, deviceID := range podDeviceIDs {
+			selectedDeviceIDs = append(selectedDeviceIDs, deviceID)
+			selectedMasters = append(selectedMasters, "")
+		}
+	}
 
 	return NICSelectResponse{
 		DeviceIDs: selectedDeviceIDs,