scheduler: numa-aware scheduling supports selecting numa node by score (

#1726)

Signed-off-by: Joseph <[email protected]>

pkg/scheduler/apis/config/types.go

@@ -103,8 +103,14 @@ type ScoringStrategy struct {
 type NodeNUMAResourceArgs struct {
 	metav1.TypeMeta
 
+	// DefaultCPUBindPolicy represents the default CPU bind policy.
+	// If it is empty and the Pod does not declare a binding policy,
+	// the core will not be bound to the LSE/LSR type Pod.
 	DefaultCPUBindPolicy CPUBindPolicy
-	ScoringStrategy      *ScoringStrategy
+	// ScoringStrategy is used to configure the scoring strategy of the Node-level.
+	ScoringStrategy *ScoringStrategy
+	// NUMAScoringStrategy is used to configure the scoring strategy of the NUMANode-level
+	NUMAScoringStrategy *ScoringStrategy
 }
 
 // CPUBindPolicy defines the CPU binding policy
@@ -132,7 +138,7 @@ const (
 	CPUExclusivePolicyNUMANodeLevel CPUExclusivePolicy = extension.CPUExclusivePolicyNUMANodeLevel
 )
 
-// NUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes
+// NUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes during binding CPUs
 type NUMAAllocateStrategy = extension.NUMAAllocateStrategy
 
 const (

pkg/scheduler/apis/config/v1beta2/defaults.go

@@ -118,6 +118,21 @@ func SetDefaults_NodeNUMAResourceArgs(obj *NodeNUMAResourceArgs) {
 			},
 		}
 	}
+	if obj.NUMAScoringStrategy == nil {
+		obj.NUMAScoringStrategy = &ScoringStrategy{
+			Type: LeastAllocated,
+			Resources: []schedconfigv1beta2.ResourceSpec{
+				{
+					Name:   string(corev1.ResourceCPU),
+					Weight: 1,
+				},
+				{
+					Name:   string(corev1.ResourceMemory),
+					Weight: 1,
+				},
+			},
+		}
+	}
 }
 
 func SetDefaults_ReservationArgs(obj *ReservationArgs) {

pkg/scheduler/apis/config/v1beta2/types.go

@@ -98,8 +98,14 @@ type ScoringStrategy struct {
 type NodeNUMAResourceArgs struct {
 	metav1.TypeMeta
 
-	DefaultCPUBindPolicy *CPUBindPolicy   `json:"defaultCPUBindPolicy,omitempty"`
-	ScoringStrategy      *ScoringStrategy `json:"scoringStrategy,omitempty"`
+	// DefaultCPUBindPolicy represents the default CPU bind policy.
+	// If it is empty and the Pod does not declare a binding policy,
+	// the core will not be bound to the LSE/LSR type Pod.
+	DefaultCPUBindPolicy *CPUBindPolicy `json:"defaultCPUBindPolicy,omitempty"`
+	// ScoringStrategy is used to configure the scoring strategy of the node-level.
+	ScoringStrategy *ScoringStrategy `json:"scoringStrategy,omitempty"`
+	// NUMAScoringStrategy is used to configure the scoring strategy of the NUMANode-level
+	NUMAScoringStrategy *ScoringStrategy `json:"numaScoringStrategy,omitempty"`
 }
 
 // CPUBindPolicy defines the CPU binding policy
@@ -127,7 +133,7 @@ const (
 	CPUExclusivePolicyNUMANodeLevel CPUExclusivePolicy = extension.CPUExclusivePolicyNUMANodeLevel
 )
 
-// NUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes
+// NUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes during binding CPUs
 type NUMAAllocateStrategy = extension.NUMAAllocateStrategy
 
 const (

pkg/scheduler/frameworkext/topologymanager/policy.go

@@ -36,6 +36,9 @@ type NUMATopologyHint struct {
 	// Preferred is set to true when the NUMANodeAffinity encodes a preferred
 	// allocation for the Pod. It is set to false otherwise.
 	Preferred bool
+	// Score is the weight of this hint. For the same Affinity,
+	// the one with higher weight will be used first.
+	Score int64
 }
 
 // IsEqual checks if NUMATopologyHint are equal
@@ -85,7 +88,7 @@ func mergePermutation(numaNodes []int, permutation []NUMATopologyHint) NUMATopol
 	mergedAffinity := bitmask.And(defaultAffinity, numaAffinities...)
 	// Build a mergedHint from the merged affinity mask, indicating if an
 	// preferred allocation was used to generate the affinity mask or not.
-	return NUMATopologyHint{mergedAffinity, preferred}
+	return NUMATopologyHint{mergedAffinity, preferred, 0}
 }
 
 func filterProvidersHints(providersHints []map[string][]NUMATopologyHint) [][]NUMATopologyHint {
@@ -97,21 +100,21 @@ func filterProvidersHints(providersHints []map[string][]NUMATopologyHint) [][]NU
 		// If hints is nil, insert a single, preferred any-numa hint into allProviderHints.
 		if len(hints) == 0 {
 			klog.V(5).Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with any resource")
-			allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, true}})
+			allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, true, 0}})
 			continue
 		}
 
 		// Otherwise, accumulate the hints for each resource type into allProviderHints.
 		for resource := range hints {
 			if hints[resource] == nil {
 				klog.V(5).Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with resource '%s'", resource)
-				allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, true}})
+				allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, true, 0}})
 				continue
 			}
 
 			if len(hints[resource]) == 0 {
 				klog.V(5).Infof("[topologymanager] Hint Provider has no possible NUMA affinities for resource '%s'", resource)
-				allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, false}})
+				allProviderHints = append(allProviderHints, []NUMATopologyHint{{nil, false, 0}})
 				continue
 			}
 
@@ -129,7 +132,7 @@ func mergeFilteredHints(numaNodes []int, filteredHints [][]NUMATopologyHint) NUM
 	// Set the bestHint to return from this function as {nil false}.
 	// This will only be returned if no better hint can be found when
 	// merging hints from each hint provider.
-	bestHint := NUMATopologyHint{defaultAffinity, false}
+	bestHint := NUMATopologyHint{defaultAffinity, false, 0}
 	iterateAllProviderTopologyHints(filteredHints, func(permutation []NUMATopologyHint) {
 		// Get the NUMANodeAffinity from each hint in the permutation and see if any
 		// of them encode unpreferred allocations.
@@ -140,6 +143,14 @@ func mergeFilteredHints(numaNodes []int, filteredHints [][]NUMATopologyHint) NUM
 			return
 		}
 
+		for _, v := range permutation {
+			if v.NUMANodeAffinity != nil && mergedHint.NUMANodeAffinity.IsEqual(v.NUMANodeAffinity) {
+				if v.Score > mergedHint.Score {
+					mergedHint.Score = v.Score
+				}
+			}
+		}
+
 		// If the current bestHint is non-preferred and the new mergedHint is
 		// preferred, always choose the preferred hint over the non-preferred one.
 		if mergedHint.Preferred && !bestHint.Preferred {
@@ -158,6 +169,11 @@ func mergeFilteredHints(numaNodes []int, filteredHints [][]NUMATopologyHint) NUM
 		// mergedHints that have a narrower NUMANodeAffinity than the
 		// NUMANodeAffinity in the current bestHint.
 		if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
+			if mergedHint.NUMANodeAffinity.Count() == bestHint.NUMANodeAffinity.Count() {
+				if mergedHint.Score > bestHint.Score {
+					bestHint = mergedHint
+				}
+			}
 			return
 		}
 

pkg/scheduler/frameworkext/topologymanager/policy_best_effort_test.go

@@ -29,12 +29,12 @@ func TestPolicyBestEffortCanAdmitPodResult(t *testing.T) {
 	}{
 		{
 			name:     "Preferred is set to false in topology hints",
-			hint:     NUMATopologyHint{nil, false},
+			hint:     NUMATopologyHint{nil, false, 0},
 			expected: true,
 		},
 		{
 			name:     "Preferred is set to true in topology hints",
-			hint:     NUMATopologyHint{nil, true},
+			hint:     NUMATopologyHint{nil, true, 0},
 			expected: true,
 		},
 	}

pkg/scheduler/frameworkext/topologymanager/policy_none_test.go

@@ -47,12 +47,12 @@ func TestPolicyNoneCanAdmitPodResult(t *testing.T) {
 	}{
 		{
 			name:     "Preferred is set to false in topology hints",
-			hint:     NUMATopologyHint{nil, false},
+			hint:     NUMATopologyHint{nil, false, 0},
 			expected: true,
 		},
 		{
 			name:     "Preferred is set to true in topology hints",
-			hint:     NUMATopologyHint{nil, true},
+			hint:     NUMATopologyHint{nil, true, 0},
 			expected: true,
 		},
 	}

pkg/scheduler/frameworkext/topologymanager/policy_restricted_test.go

@@ -47,12 +47,12 @@ func TestPolicyRestrictedCanAdmitPodResult(t *testing.T) {
 	}{
 		{
 			name:     "Preferred is set to false in topology hints",
-			hint:     NUMATopologyHint{nil, false},
+			hint:     NUMATopologyHint{nil, false, 0},
 			expected: false,
 		},
 		{
 			name:     "Preferred is set to true in topology hints",
-			hint:     NUMATopologyHint{nil, true},
+			hint:     NUMATopologyHint{nil, true, 0},
 			expected: true,
 		},
 	}

pkg/scheduler/frameworkext/topologymanager/policy_single_numa_node.go

@@ -64,13 +64,13 @@ func filterSingleNumaHints(allResourcesHints [][]NUMATopologyHint) [][]NUMATopol
 
 func (p *singleNumaNodePolicy) Merge(providersHints []map[string][]NUMATopologyHint) (NUMATopologyHint, bool) {
 	filteredHints := filterProvidersHints(providersHints)
-	// Filter to only include don't cares and hints with a single NUMA node.
+	// Filter to only include don't care and hints with a single NUMA node.
 	singleNumaHints := filterSingleNumaHints(filteredHints)
 	bestHint := mergeFilteredHints(p.numaNodes, singleNumaHints)
 
 	defaultAffinity, _ := bitmask.NewBitMask(p.numaNodes...)
 	if bestHint.NUMANodeAffinity.IsEqual(defaultAffinity) {
-		bestHint = NUMATopologyHint{nil, bestHint.Preferred}
+		bestHint = NUMATopologyHint{nil, bestHint.Preferred, 0}
 	}
 
 	admit := p.canAdmitPodResult(&bestHint)

pkg/scheduler/frameworkext/topologymanager/policy_single_numa_node_test.go

@@ -30,7 +30,7 @@ func TestPolicySingleNumaNodeCanAdmitPodResult(t *testing.T) {
 	}{
 		{
 			name:     "Preferred is set to false in topology hints",
-			hint:     NUMATopologyHint{nil, false},
+			hint:     NUMATopologyHint{nil, false, 0},
 			expected: false,
 		},
 	}

pkg/scheduler/plugins/nodenumaresource/plugin.go

@@ -74,6 +74,7 @@ type Plugin struct {
 	pluginArgs      *schedulingconfig.NodeNUMAResourceArgs
 	nrtLister       topologylister.NodeResourceTopologyLister
 	scorer          *resourceAllocationScorer
+	numaScorer      *resourceAllocationScorer
 	resourceManager ResourceManager
 
 	topologyOptionsManager TopologyOptionsManager
@@ -114,6 +115,14 @@ func NewWithOptions(args runtime.Object, handle framework.Handle, opts ...Option
 	if !exists {
 		return nil, fmt.Errorf("scoring strategy %s is not supported", strategy)
 	}
+	scorer := scorePlugin(pluginArgs)
+
+	strategy = pluginArgs.NUMAScoringStrategy.Type
+	scorePlugin, exists = resourceStrategyTypeMap[strategy]
+	if !exists {
+		return nil, fmt.Errorf("numa scoring strategy %s is not supported", strategy)
+	}
+	numaScorer := scorePlugin(pluginArgs)
 
 	options := &pluginOptions{}
 	for _, optFnc := range opts {
@@ -144,7 +153,8 @@ func NewWithOptions(args runtime.Object, handle framework.Handle, opts ...Option
 		handle:                 handle,
 		pluginArgs:             pluginArgs,
 		nrtLister:              nrtLister,
-		scorer:                 scorePlugin(pluginArgs),
+		scorer:                 scorer,
+		numaScorer:             numaScorer,
 		resourceManager:        options.resourceManager,
 		topologyOptionsManager: options.topologyOptionsManager,
 	}, nil