diff --git a/pkg/features/koordlet_features.go b/pkg/features/koordlet_features.go index cb14d7cd6..cb76621ac 100644 --- a/pkg/features/koordlet_features.go +++ b/pkg/features/koordlet_features.go @@ -133,6 +133,14 @@ const ( // // ColdPageCollector enables coldPageCollector feature of koordlet. ColdPageCollector featuregate.Feature = "ColdPageCollector" + + // HugePageReport enables hugepage collector feature of koordlet. + // This feature supports reporting of hugepages. + // The koord-scheduler will allocate hugepage information based on the user's hugepage request and add it to the Pod's annotations. + // Format: scheduling.koordinator.sh/resource-status: '{"numaNodeResources":[{"node":1,"resources":{"hugepages-1Gi":"50Gi"}}]}'. + // Backend applications can enable the hugepages based on the allocation results. + // For example, the CSI mounts the pre-allocated hugepages into the pod. + HugePageReport featuregate.Feature = "HugePageReport" ) func init() { @@ -161,6 +169,7 @@ var ( PSICollector: {Default: false, PreRelease: featuregate.Alpha}, BlkIOReconcile: {Default: false, PreRelease: featuregate.Alpha}, ColdPageCollector: {Default: false, PreRelease: featuregate.Alpha}, + HugePageReport: {Default: false, PreRelease: featuregate.Alpha}, } ) diff --git a/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector.go b/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector.go index d894bb6ce..22c793382 100644 --- a/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector.go +++ b/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "github.com/koordinator-sh/koordinator/pkg/features" "github.com/koordinator-sh/koordinator/pkg/koordlet/metriccache" "github.com/koordinator-sh/koordinator/pkg/koordlet/metrics" "github.com/koordinator-sh/koordinator/pkg/koordlet/metricsadvisor/framework" @@ -111,6 +112,9 @@ func (n *nodeInfoCollector) collectNodeNUMAInfo() error { metrics.RecordCollectNodeNUMAInfoStatus(err) return err } + if features.DefaultKoordletFeatureGate.Enabled(features.HugePageReport) { + koordletutil.GetAndMergeHugepageToNumaInfo(nodeNUMAInfo) + } klog.V(6).Infof("collect NUMA info successfully, info %+v", nodeNUMAInfo) n.storage.Set(metriccache.NodeNUMAInfoKey, nodeNUMAInfo) diff --git a/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector_test.go b/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector_test.go index f87921114..363d343e4 100644 --- a/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector_test.go +++ b/pkg/koordlet/metricsadvisor/collectors/nodeinfo/node_info_collector_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/atomic" + "github.com/koordinator-sh/koordinator/pkg/features" "github.com/koordinator-sh/koordinator/pkg/koordlet/metriccache" "github.com/koordinator-sh/koordinator/pkg/koordlet/metricsadvisor/framework" koordletutil "github.com/koordinator-sh/koordinator/pkg/koordlet/util" @@ -63,17 +64,6 @@ func TestNodeInfoCollector(t *testing.T) { } func Test_collectNodeNUMAInfo(t *testing.T) { - helper := system.NewFileTestUtil(t) - defer helper.Cleanup() - metricCache, err := metriccache.NewMetricCache(&metriccache.Config{ - TSDBPath: t.TempDir(), - TSDBEnablePromMetrics: false, - }) - assert.NoError(t, err) - defer func() { - err = metricCache.Close() - assert.NoError(t, err) - }() numaMemInfoContentStr0 := `Node 0 MemTotal: 263432804 kB Node 0 MemFree: 254391744 kB @@ -147,10 +137,7 @@ Node 1 HugePages_Total: 0 Node 1 HugePages_Free: 0 Node 1 HugePages_Rsvd: 0 Node 1 HugePages_Surp: 0` - numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") - helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) - numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") - helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + testMemInfo0 := &koordletutil.MemInfo{ MemTotal: 263432804, MemFree: 254391744, MemAvailable: 256703236, Buffers: 958096, Cached: 0, SwapCached: 0, @@ -179,41 +166,336 @@ Node 1 HugePages_Surp: 0` HugePages_Total: 0, HugePages_Free: 0, HugePages_Rsvd: 0, HugePages_Surp: 0, } - expected := &koordletutil.NodeNUMAInfo{ - NUMAInfos: []koordletutil.NUMAInfo{ - { - NUMANodeID: 0, - MemInfo: testMemInfo0, + + tts := []struct { + name string + initfunc func(*system.FileTestUtil) + enableHugePage bool + expected *koordletutil.NodeNUMAInfo + }{ + { + name: "only memory", + initfunc: func(helper *system.FileTestUtil) { + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + }, + enableHugePage: false, + expected: &koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + }, + }, + { + name: "memory and hugepage, but featuregate not enable", + initfunc: func(helper *system.FileTestUtil) { + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + numaHugePage2MPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage2MPath0, "20") + numaHugePage2MPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage2MPath1, "40") + }, + enableHugePage: false, + expected: &koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + }, + }, + { + name: "memory and 2M hugepage", + initfunc: func(helper *system.FileTestUtil) { + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + numaHugePage2MPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath0, "20") + numaHugePage2MPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath1, "40") }, - { - NUMANodeID: 1, - MemInfo: testMemInfo1, + enableHugePage: true, + expected: &koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 0, + }, + }, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 40, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 0, + }, + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 0, + }, + }, + 1: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 40, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 0, + }, + }, + }, }, }, - MemInfoMap: map[int32]*koordletutil.MemInfo{ - 0: testMemInfo0, - 1: testMemInfo1, + { + name: "memory and 1G hugepage", + initfunc: func(helper *system.FileTestUtil) { + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + numaHugePage1GPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath0, "10") + + numaHugePage1GPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath1, "20") + }, + enableHugePage: true, + expected: &koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 0, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 0, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 20, + }, + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 0, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + 1: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 0, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 20, + }, + }, + }, + }, + }, + { + name: "memory and 2M & 1G hugepage", + initfunc: func(helper *system.FileTestUtil) { + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + numaHugePage1GPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath0, "10") + numaHugePage2MPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath0, "20") + + numaHugePage1GPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath1, "10") + numaHugePage2MPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath1, "20") + }, + enableHugePage: true, + expected: &koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + 1: { + koordletutil.Hugepage2Mkbyte: { + PageSize: koordletutil.Hugepage2Mkbyte, + NumPages: 20, + }, + koordletutil.Hugepage1Gkbyte: { + PageSize: koordletutil.Hugepage1Gkbyte, + NumPages: 10, + }, + }, + }, + }, }, } - t.Run("test", func(t *testing.T) { - c := &nodeInfoCollector{ - collectInterval: 60 * time.Second, - storage: metricCache, - started: atomic.NewBool(false), - } + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() - // test collect successfully - err = c.collectNodeNUMAInfo() - assert.NoError(t, err) - nodeNUMAInfoRaw, ok := c.storage.Get(metriccache.NodeNUMAInfoKey) - assert.True(t, ok) - nodeNUMAInfo, ok := nodeNUMAInfoRaw.(*koordletutil.NodeNUMAInfo) - assert.True(t, ok) - assert.Equal(t, expected, nodeNUMAInfo) - - // test collect failed when sys files are missing - helper.Cleanup() - err = c.collectNodeNUMAInfo() - assert.Error(t, err) - }) + enabled := features.DefaultKoordletFeatureGate.Enabled(features.HugePageReport) + testFeatureGates := map[string]bool{string(features.HugePageReport): tt.enableHugePage} + err := features.DefaultMutableKoordletFeatureGate.SetFromMap(testFeatureGates) + assert.NoError(t, err) + defer func() { + testFeatureGates[string(features.HugePageReport)] = enabled + err = features.DefaultMutableKoordletFeatureGate.SetFromMap(testFeatureGates) + assert.NoError(t, err) + }() + + metricCache, err := metriccache.NewMetricCache(&metriccache.Config{ + TSDBPath: t.TempDir(), + TSDBEnablePromMetrics: false, + }) + assert.NoError(t, err) + defer func() { + err = metricCache.Close() + assert.NoError(t, err) + }() + + tt.initfunc(helper) + + c := &nodeInfoCollector{ + collectInterval: 60 * time.Second, + storage: metricCache, + started: atomic.NewBool(false), + } + err = c.collectNodeNUMAInfo() + assert.NoError(t, err) + nodeNUMAInfoRaw, ok := c.storage.Get(metriccache.NodeNUMAInfoKey) + assert.True(t, ok) + nodeNUMAInfo, ok := nodeNUMAInfoRaw.(*koordletutil.NodeNUMAInfo) + assert.True(t, ok) + assert.Equal(t, tt.expected, nodeNUMAInfo) + + // test collect failed when sys files are missing + helper.Cleanup() + err = c.collectNodeNUMAInfo() + assert.Error(t, err) + }) + } } diff --git a/pkg/koordlet/statesinformer/impl/states_noderesourcetopology.go b/pkg/koordlet/statesinformer/impl/states_noderesourcetopology.go index 6abeacc7f..a79ae85c7 100644 --- a/pkg/koordlet/statesinformer/impl/states_noderesourcetopology.go +++ b/pkg/koordlet/statesinformer/impl/states_noderesourcetopology.go @@ -766,7 +766,7 @@ func (s *nodeTopoInformer) calTopologyZoneList(nodeCPUInfo *metriccache.NodeCPUI } nodeNUMAInfo, ok := nodeNUMAInfoRaw.(*koordletutil.NodeNUMAInfo) if !ok { - klog.Fatalf("type error, expect %T, but got %T", koordletutil.NodeNUMAInfo{}, nodeNUMAInfoRaw) + klog.Fatalf("type error, expect %T, but got %T", koordletutil.NodeNUMAInfo{}, nodeNUMAInfoRaw) } nodeNum := len(nodeNUMAInfo.NUMAInfos) @@ -785,18 +785,42 @@ func (s *nodeTopoInformer) calTopologyZoneList(nodeCPUInfo *metriccache.NodeCPUI } else { cpuQuant = resource.MustParse("0") } + + hugepage2Mbyte := uint64(0) + hugepage1Gbyte := uint64(0) + hugepage2MQuant := *resource.NewQuantity(0, resource.BinarySI) + hugepage1GQuant := *resource.NewQuantity(0, resource.BinarySI) + + if features.DefaultKoordletFeatureGate.Enabled(features.HugePageReport) { + hugepageInfos, ok := nodeNUMAInfo.HugePagesMap[int32(i)] + if ok { + if _, ok := hugepageInfos[koordletutil.Hugepage2Mkbyte]; ok { + hugepage2Mbyte = hugepageInfos[koordletutil.Hugepage2Mkbyte].MemTotalBytes() + hugepage2MQuant = *resource.NewQuantity(int64(hugepage2Mbyte), resource.BinarySI) + } + if _, ok := hugepageInfos[koordletutil.Hugepage1Gkbyte]; ok { + hugepage1Gbyte = hugepageInfos[koordletutil.Hugepage1Gkbyte].MemTotalBytes() + hugepage1GQuant = *resource.NewQuantity(int64(hugepage1Gbyte), resource.BinarySI) + } + } + } + var memQuant resource.Quantity memInfo, ok := nodeNUMAInfo.MemInfoMap[int32(i)] if ok { - memQuant = *resource.NewQuantity(int64(memInfo.MemTotalBytes()), resource.BinarySI) + memQuant = *resource.NewQuantity(int64(memInfo.MemTotalBytes()-hugepage2Mbyte-hugepage1Gbyte), resource.BinarySI) } else { - memQuant = resource.MustParse("0") + memQuant = *resource.NewQuantity(0, resource.BinarySI) } + zoneName := util.GenNodeZoneName(i) zoneResourceList[zoneName] = corev1.ResourceList{ - corev1.ResourceCPU: cpuQuant, - corev1.ResourceMemory: memQuant, + corev1.ResourceCPU: cpuQuant, + corev1.ResourceMemory: memQuant, + corev1.ResourceHugePagesPrefix + "2Mi": hugepage2MQuant, + corev1.ResourceHugePagesPrefix + "1Gi": hugepage1GQuant, } + } zoneList := util.ZoneResourceListToZoneList(zoneResourceList) diff --git a/pkg/koordlet/statesinformer/impl/states_noderesourcetopology_test.go b/pkg/koordlet/statesinformer/impl/states_noderesourcetopology_test.go index 329831c0a..7da063db1 100644 --- a/pkg/koordlet/statesinformer/impl/states_noderesourcetopology_test.go +++ b/pkg/koordlet/statesinformer/impl/states_noderesourcetopology_test.go @@ -491,6 +491,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(4, resource.DecimalSI), Available: *resource.NewQuantity(4, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269755191296, resource.BinarySI), @@ -509,6 +521,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(4, resource.DecimalSI), Available: *resource.NewQuantity(4, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269754368000, resource.BinarySI), @@ -585,6 +609,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(1, resource.DecimalSI), Available: *resource.NewQuantity(1, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269755191296, resource.BinarySI), @@ -609,6 +645,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(1, resource.DecimalSI), Available: *resource.NewQuantity(1, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269754368000, resource.BinarySI), @@ -639,6 +687,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(1, resource.DecimalSI), Available: *resource.NewQuantity(1, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269755191296, resource.BinarySI), @@ -663,6 +723,18 @@ func Test_reportNodeTopology(t *testing.T) { Allocatable: *resource.NewQuantity(1, resource.DecimalSI), Available: *resource.NewQuantity(1, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", Capacity: *resource.NewQuantity(269754368000, resource.BinarySI), @@ -1212,11 +1284,12 @@ func Test_calTopologyZoneList(t *testing.T) { nodeCPUInfo *metriccache.NodeCPUInfo } tests := []struct { - name string - fields fields - args args - want topologyv1alpha1.ZoneList - wantErr bool + name string + fields fields + args args + hugepageEnable bool + want topologyv1alpha1.ZoneList + wantErr bool }{ { name: "err when numa info not exist", @@ -1284,13 +1357,114 @@ func Test_calTopologyZoneList(t *testing.T) { { NUMANodeID: 0, MemInfo: &koordletutil.MemInfo{ - MemTotal: 1024000, + MemTotal: 157286400, // 150G + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: { + MemTotal: 157286400, // 150G + }, + }, + }, true).Times(1) + return mc + }, + }, + args: args{ + nodeCPUInfo: &metriccache.NodeCPUInfo{ + TotalInfo: koordletutil.CPUTotalInfo{ + NodeToCPU: map[int32][]koordletutil.ProcessorInfo{ + 0: { + { + CPUID: 0, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 1, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + }, + }, + }, + }, + }, + want: topologyv1alpha1.ZoneList{ + { + Name: "node-0", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(2, resource.DecimalSI), + Allocatable: *resource.NewQuantity(2, resource.DecimalSI), + Available: *resource.NewQuantity(2, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "calculate single numa node with hugepage, but featuregate hugepage not enable", + fields: fields{ + metricCache: func(ctrl *gomock.Controller) metriccache.MetricCache { + mc := mock_metriccache.NewMockMetricCache(ctrl) + mc.EXPECT().Get(metriccache.NodeNUMAInfoKey).Return(&koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G }, }, }, MemInfoMap: map[int32]*koordletutil.MemInfo{ 0: { - MemTotal: 1024000, + MemTotal: 157286400, // 150G + }, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G }, }, }, true).Times(1) @@ -1330,11 +1504,125 @@ func Test_calTopologyZoneList(t *testing.T) { Allocatable: *resource.NewQuantity(2, resource.DecimalSI), Available: *resource.NewQuantity(2, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", - Capacity: *resource.NewQuantity(1048576000, resource.BinarySI), - Allocatable: *resource.NewQuantity(1048576000, resource.BinarySI), - Available: *resource.NewQuantity(1048576000, resource.BinarySI), + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "calculate single numa node with hugepage, featuregate hugepage enable", + hugepageEnable: true, + fields: fields{ + metricCache: func(ctrl *gomock.Controller) metriccache.MetricCache { + mc := mock_metriccache.NewMockMetricCache(ctrl) + mc.EXPECT().Get(metriccache.NodeNUMAInfoKey).Return(&koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: { + MemTotal: 157286400, // 150G + }, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G + }, + }, + }, true).Times(1) + return mc + }, + }, + args: args{ + nodeCPUInfo: &metriccache.NodeCPUInfo{ + TotalInfo: koordletutil.CPUTotalInfo{ + NodeToCPU: map[int32][]koordletutil.ProcessorInfo{ + 0: { + { + CPUID: 0, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 1, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + }, + }, + }, + }, + }, + want: topologyv1alpha1.ZoneList{ + { + Name: "node-0", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(2, resource.DecimalSI), + Allocatable: *resource.NewQuantity(2, resource.DecimalSI), + Available: *resource.NewQuantity(2, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(32212254720, resource.BinarySI), + Allocatable: *resource.NewQuantity(32212254720, resource.BinarySI), + Available: *resource.NewQuantity(32212254720, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(62914560, resource.BinarySI), + Allocatable: *resource.NewQuantity(62914560, resource.BinarySI), + Available: *resource.NewQuantity(62914560, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(128786104320, resource.BinarySI), + Allocatable: *resource.NewQuantity(128786104320, resource.BinarySI), + Available: *resource.NewQuantity(128786104320, resource.BinarySI), }, }, }, @@ -1351,22 +1639,220 @@ func Test_calTopologyZoneList(t *testing.T) { { NUMANodeID: 0, MemInfo: &koordletutil.MemInfo{ - MemTotal: 1024000, + MemTotal: 157286400, // 150G + }, + }, + { + NUMANodeID: 1, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: { + MemTotal: 157286400, + }, + 1: { + MemTotal: 157286400, + }, + }, + }, true).Times(1) + return mc + }, + }, + args: args{ + nodeCPUInfo: &metriccache.NodeCPUInfo{ + TotalInfo: koordletutil.CPUTotalInfo{ + NodeToCPU: map[int32][]koordletutil.ProcessorInfo{ + 0: { + { + CPUID: 0, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 1, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 4, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 5, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + }, + 1: { + { + CPUID: 2, + CoreID: 2, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 3, + CoreID: 3, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 6, + CoreID: 2, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 7, + CoreID: 3, + SocketID: 1, + NodeID: 1, + }, + }, + }, + }, + }, + }, + want: topologyv1alpha1.ZoneList{ + { + Name: "node-0", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(4, resource.DecimalSI), + Allocatable: *resource.NewQuantity(4, resource.DecimalSI), + Available: *resource.NewQuantity(4, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), + }, + }, + }, + { + Name: "node-1", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(4, resource.DecimalSI), + Allocatable: *resource.NewQuantity(4, resource.DecimalSI), + Available: *resource.NewQuantity(4, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "calculate multiple numa nodes with hugepage, but featuregate hugepage not enable", + fields: fields{ + metricCache: func(ctrl *gomock.Controller) metriccache.MetricCache { + mc := mock_metriccache.NewMockMetricCache(ctrl) + mc.EXPECT().Get(metriccache.NodeNUMAInfoKey).Return(&koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G }, }, { NUMANodeID: 1, MemInfo: &koordletutil.MemInfo{ - MemTotal: 1000000, + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G }, }, }, MemInfoMap: map[int32]*koordletutil.MemInfo{ 0: { - MemTotal: 1024000, + MemTotal: 157286400, }, 1: { - MemTotal: 1000000, + MemTotal: 157286400, + }, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, + }, + 1: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, }, }, }, true).Times(1) @@ -1444,11 +1930,23 @@ func Test_calTopologyZoneList(t *testing.T) { Allocatable: *resource.NewQuantity(4, resource.DecimalSI), Available: *resource.NewQuantity(4, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", - Capacity: *resource.NewQuantity(1048576000, resource.BinarySI), - Allocatable: *resource.NewQuantity(1048576000, resource.BinarySI), - Available: *resource.NewQuantity(1048576000, resource.BinarySI), + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), }, }, }, @@ -1462,11 +1960,222 @@ func Test_calTopologyZoneList(t *testing.T) { Allocatable: *resource.NewQuantity(4, resource.DecimalSI), Available: *resource.NewQuantity(4, resource.DecimalSI), }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(0, resource.BinarySI), + Allocatable: *resource.NewQuantity(0, resource.BinarySI), + Available: *resource.NewQuantity(0, resource.BinarySI), + }, { Name: "memory", - Capacity: *resource.NewQuantity(1024000000, resource.BinarySI), - Allocatable: *resource.NewQuantity(1024000000, resource.BinarySI), - Available: *resource.NewQuantity(1024000000, resource.BinarySI), + Capacity: *resource.NewQuantity(161061273600, resource.BinarySI), + Allocatable: *resource.NewQuantity(161061273600, resource.BinarySI), + Available: *resource.NewQuantity(161061273600, resource.BinarySI), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "calculate multiple numa nodes with hugepage, featuregate hugepage enable", + hugepageEnable: true, + fields: fields{ + metricCache: func(ctrl *gomock.Controller) metriccache.MetricCache { + mc := mock_metriccache.NewMockMetricCache(ctrl) + mc.EXPECT().Get(metriccache.NodeNUMAInfoKey).Return(&koordletutil.NodeNUMAInfo{ + NUMAInfos: []koordletutil.NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G + }, + }, + { + NUMANodeID: 1, + MemInfo: &koordletutil.MemInfo{ + MemTotal: 157286400, // 150G + }, + HugePages: map[uint64]*koordletutil.HugePagesInfo{ + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, // 60M + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, // 30G + }, + }, + }, + MemInfoMap: map[int32]*koordletutil.MemInfo{ + 0: { + MemTotal: 157286400, + }, + 1: { + MemTotal: 157286400, + }, + }, + HugePagesMap: map[int32]map[uint64]*koordletutil.HugePagesInfo{ + 0: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, + }, + 1: { + koordletutil.Hugepage2Mkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage2Mkbyte, + }, + koordletutil.Hugepage1Gkbyte: { + NumPages: 30, + PageSize: koordletutil.Hugepage1Gkbyte, + }, + }, + }, + }, true).Times(1) + return mc + }, + }, + args: args{ + nodeCPUInfo: &metriccache.NodeCPUInfo{ + TotalInfo: koordletutil.CPUTotalInfo{ + NodeToCPU: map[int32][]koordletutil.ProcessorInfo{ + 0: { + { + CPUID: 0, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 1, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 4, + CoreID: 0, + SocketID: 0, + NodeID: 0, + }, + { + CPUID: 5, + CoreID: 1, + SocketID: 0, + NodeID: 0, + }, + }, + 1: { + { + CPUID: 2, + CoreID: 2, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 3, + CoreID: 3, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 6, + CoreID: 2, + SocketID: 1, + NodeID: 1, + }, + { + CPUID: 7, + CoreID: 3, + SocketID: 1, + NodeID: 1, + }, + }, + }, + }, + }, + }, + want: topologyv1alpha1.ZoneList{ + { + Name: "node-0", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(4, resource.DecimalSI), + Allocatable: *resource.NewQuantity(4, resource.DecimalSI), + Available: *resource.NewQuantity(4, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(32212254720, resource.BinarySI), + Allocatable: *resource.NewQuantity(32212254720, resource.BinarySI), + Available: *resource.NewQuantity(32212254720, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(62914560, resource.BinarySI), + Allocatable: *resource.NewQuantity(62914560, resource.BinarySI), + Available: *resource.NewQuantity(62914560, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(128786104320, resource.BinarySI), + Allocatable: *resource.NewQuantity(128786104320, resource.BinarySI), + Available: *resource.NewQuantity(128786104320, resource.BinarySI), + }, + }, + }, + { + Name: "node-1", + Type: util.NodeZoneType, + Resources: topologyv1alpha1.ResourceInfoList{ + { + Name: "cpu", + Capacity: *resource.NewQuantity(4, resource.DecimalSI), + Allocatable: *resource.NewQuantity(4, resource.DecimalSI), + Available: *resource.NewQuantity(4, resource.DecimalSI), + }, + { + Name: "hugepages-1Gi", + Capacity: *resource.NewQuantity(32212254720, resource.BinarySI), + Allocatable: *resource.NewQuantity(32212254720, resource.BinarySI), + Available: *resource.NewQuantity(32212254720, resource.BinarySI), + }, + { + Name: "hugepages-2Mi", + Capacity: *resource.NewQuantity(62914560, resource.BinarySI), + Allocatable: *resource.NewQuantity(62914560, resource.BinarySI), + Available: *resource.NewQuantity(62914560, resource.BinarySI), + }, + { + Name: "memory", + Capacity: *resource.NewQuantity(128786104320, resource.BinarySI), + Allocatable: *resource.NewQuantity(128786104320, resource.BinarySI), + Available: *resource.NewQuantity(128786104320, resource.BinarySI), }, }, }, @@ -1479,6 +2188,16 @@ func Test_calTopologyZoneList(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() + enabled := features.DefaultKoordletFeatureGate.Enabled(features.HugePageReport) + testFeatureGates := map[string]bool{string(features.HugePageReport): tt.hugepageEnable} + err := features.DefaultMutableKoordletFeatureGate.SetFromMap(testFeatureGates) + assert.NoError(t, err) + defer func() { + testFeatureGates[string(features.HugePageReport)] = enabled + err = features.DefaultMutableKoordletFeatureGate.SetFromMap(testFeatureGates) + assert.NoError(t, err) + }() + s := &nodeTopoInformer{ metricCache: tt.fields.metricCache(ctrl), } diff --git a/pkg/koordlet/util/meminfo.go b/pkg/koordlet/util/meminfo.go index b4dd8edae..7bdcd3fb9 100644 --- a/pkg/koordlet/util/meminfo.go +++ b/pkg/koordlet/util/meminfo.go @@ -29,6 +29,11 @@ import ( "github.com/koordinator-sh/koordinator/pkg/koordlet/util/system" ) +const ( + Hugepage1Gkbyte = 1048576 + Hugepage2Mkbyte = 2048 +) + // MemInfo is the content of system /proc/meminfo. // NOTE: the unit of each field is KiB. type MemInfo struct { @@ -78,6 +83,15 @@ type MemInfo struct { DirectMap1G uint64 `json:"direct_map_1G"` } +type HugePagesInfo struct { + NumPages uint64 `json:"numPages,omitempty"` + PageSize uint64 `json:"pageSize,omitempty"` +} + +func (i *HugePagesInfo) MemTotalBytes() uint64 { + return i.PageSize * i.NumPages * 1024 +} + // MemTotalBytes returns the mem info's total bytes. func (i *MemInfo) MemTotalBytes() uint64 { return i.MemTotal * 1024 @@ -148,6 +162,68 @@ func readMemInfo(path string, isNUMA bool) (*MemInfo, error) { return &info, nil } +func GetHugePagesInfo(nodeDir string) (map[uint64]*HugePagesInfo, error) { + hugePagesInfo := map[uint64]*HugePagesInfo{ + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + } + hugepageDir := system.GetNUMAHugepagesDir(nodeDir) + hugeDirs, err := os.ReadDir(hugepageDir) + if err != nil { + klog.Warningf("failed to read hugepage dir %s, err: %w", hugepageDir, err) + return hugePagesInfo, err + } + + for _, st := range hugeDirs { + nameArray := strings.Split(st.Name(), "-") + if len(nameArray) < 2 { + klog.Warningf("Split '-' failed, the directory name %s is invalid, it must be either hugepages-1048576kB or hugepages-2048kB, nameArray: %v, len(nameArray): %d", st.Name(), nameArray, len(nameArray)) + continue + } + pageSizeArray := strings.Split(nameArray[1], "kB") + if len(pageSizeArray) < 1 { + klog.Warningf("Split 'kB' failed, the directory name %s is invalid, it must be either hugepages-1048576kB or hugepages-2048kB, pageSizeArray: %v, len(pageSizeArray): %d", st.Name(), pageSizeArray, len(pageSizeArray)) + continue + } + pageSize, err := strconv.ParseUint(pageSizeArray[0], 10, 64) + if err != nil { + klog.Warningf("hugepage pageSize parse failed, it must be either hugepages-1048576kB or hugepages-2048kB, the dir: %s, origin data: %s, err: %v", st.Name(), pageSizeArray[0], err) + continue + } + + if _, ok := hugePagesInfo[pageSize]; !ok { + klog.Warningf("An abnormal hugepage %d, the dir: %s", pageSize, st.Name()) + continue + } + + nrPath := system.GetNUMAHugepagesNrPath(nodeDir, st.Name()) + val, err := os.ReadFile(nrPath) + if err != nil { + return hugePagesInfo, err + } + + var numPages uint64 + // we use sscanf as the file as a new-line that trips up ParseUint + // it returns the number of tokens successfully parsed, so if + // n != 1, it means we were unable to parse a number from the file + n, err := fmt.Sscanf(string(val), "%d", &numPages) + if err != nil || n != 1 { + klog.Warningf("could not parse file nr_hugepage for %s, contents %q", st.Name(), string(val)) + continue + } + + hugePagesInfo[pageSize].NumPages = numPages + } + + return hugePagesInfo, nil +} + func GetMemInfo() (*MemInfo, error) { memInfoPath := system.GetProcFilePath(system.ProcMemInfoName) memInfo, err := readMemInfo(memInfoPath, false) @@ -158,15 +234,17 @@ func GetMemInfo() (*MemInfo, error) { } type NUMAInfo struct { - NUMANodeID int32 `json:"numaNodeID,omitempty"` - MemInfo *MemInfo `json:"memInfo,omitempty"` + NUMANodeID int32 `json:"numaNodeID,omitempty"` + MemInfo *MemInfo `json:"memInfo,omitempty"` + HugePages map[uint64]*HugePagesInfo `json:"hugePages,omitempty"` } // NodeNUMAInfo represents the node NUMA information. // Currently, it just contains the meminfo for each NUMA node. type NodeNUMAInfo struct { - NUMAInfos []NUMAInfo `json:"numaInfos,omitempty"` - MemInfoMap map[int32]*MemInfo `json:"memInfoMap,omitempty"` // NUMANodeID -> MemInfo + NUMAInfos []NUMAInfo `json:"numaInfos,omitempty"` + MemInfoMap map[int32]*MemInfo `json:"memInfoMap,omitempty"` // NUMANodeID -> MemInfo + HugePagesMap map[int32]map[uint64]*HugePagesInfo `json:"hugePagesMap,omitempty"` } // GetNodeNUMAInfo gets the node NUMA information with the pre-configured sysfs path. @@ -229,3 +307,73 @@ func GetNodeNUMAInfo() (*NodeNUMAInfo, error) { return result, nil } + +// GetNodeHugePagesInfo gets the node NUMA hugepage information with pre-configured sysfs path. +func GetNodeHugePagesInfo() (map[int32]map[uint64]*HugePagesInfo, error) { + numaNodeParentDir := system.GetSysNUMADir() + nodeDirs, err := os.ReadDir(numaNodeParentDir) + if err != nil { + return nil, fmt.Errorf("failed to read NUMA dir, err: %w", err) + } + + hugePagesMap := make(map[int32]map[uint64]*HugePagesInfo) + for _, n := range nodeDirs { + dirName := n.Name() // assert string pattern `nodeX` + if len(dirName) < 4 || dirName[:4] != "node" { + klog.V(4).Infof("failed to get node NUMA info, err: invalid dir name %s", dirName) + return nil, fmt.Errorf("failed to get node NUMA info, err: invalid dir name %s", dirName) + } + + nodeIDRaw, err := strconv.ParseInt(dirName[4:], 10, 32) + if err != nil { + klog.V(4).Infof("failed to parse NUMA ID, err: invalid dir name %s, err %v", dirName, err) + return nil, fmt.Errorf("failed to get node NUMA info, err: invalid dir name %s", dirName) + } + nodeID := int32(nodeIDRaw) + hugepageInfos, err := GetHugePagesInfo(dirName) + if err != nil { + klog.V(4).Infof("failed to read hugepage info, just set empty hugepage info, dir %s, err: %v", dirName, err) + return nil, fmt.Errorf("failed to read hugepage info, just set empty hugepage info, dir %s, err: %v", dirName, err) + } + hugePagesMap[nodeID] = hugepageInfos + } + return hugePagesMap, nil +} + +func GetAndMergeHugepageToNumaInfo(numaInfo *NodeNUMAInfo) *NodeNUMAInfo { + var hugePagesMap map[int32]map[uint64]*HugePagesInfo + var err error + hugePagesMap, err = GetNodeHugePagesInfo() + if err != nil { + hugePagesMap = map[int32]map[uint64]*HugePagesInfo{ + 0: { + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + }, + 1: { + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + }, + } + } + numaInfo.HugePagesMap = hugePagesMap + for i, info := range numaInfo.NUMAInfos { + if _, ok := hugePagesMap[info.NUMANodeID]; ok { + numaInfo.NUMAInfos[i].HugePages = hugePagesMap[info.NUMANodeID] + } + } + + return numaInfo +} diff --git a/pkg/koordlet/util/meminfo_test.go b/pkg/koordlet/util/meminfo_test.go index b576e0e6a..73c278196 100644 --- a/pkg/koordlet/util/meminfo_test.go +++ b/pkg/koordlet/util/meminfo_test.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "fmt" "path/filepath" "testing" @@ -486,3 +487,338 @@ Node 1 HugePages_Surp: 0` assert.Error(t, err) assert.Nil(t, got) } + +func TestGetNodeHugePagesInfo(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + + numaHugePage1GPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath0, "10") + numaHugePage2MPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath0, "20") + + numaHugePage1GPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath1, "10") + numaHugePage2MPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath1, "20") + + expected := map[int32]map[uint64]*HugePagesInfo{ + 0: { + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + }, + 1: { + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + }, + } + + got, err := GetNodeHugePagesInfo() + assert.NoError(t, err) + assert.Equal(t, expected, got) + + // test partial failure + numaMemInfoPath2 := system.GetNUMAMemInfoPath("node2") + helper.MkDirAll(filepath.Dir(numaMemInfoPath2)) + got, err = GetNodeHugePagesInfo() + assert.Error(t, err) + assert.Nil(t, got) + + // test path not exist + helper.Cleanup() + got, err = GetNodeHugePagesInfo() + assert.Error(t, err) + assert.Nil(t, got) +} + +func TestGetAndMergeHugepageToNumaInfo(t *testing.T) { + helper := system.NewFileTestUtil(t) + defer helper.Cleanup() + numaMemInfoContentStr0 := `Node 0 MemTotal: 263432804 kB +Node 0 MemFree: 254391744 kB +Node 0 MemAvailable: 256703236 kB +Node 0 Buffers: 958096 kB +Node 0 Cached: 0 kB +Node 0 SwapCached: 0 kB +Node 0 Active: 2786012 kB +Node 0 Inactive: 2223752 kB +Node 0 Active(anon): 289488 kB +Node 0 Inactive(anon): 1300 kB +Node 0 Active(file): 2496524 kB +Node 0 Inactive(file): 2222452 kB +Node 0 Unevictable: 0 kB +Node 0 Mlocked: 0 kB +Node 0 SwapTotal: 0 kB +Node 0 SwapFree: 0 kB +Node 0 Dirty: 624 kB +Node 0 Writeback: 0 kB +Node 0 AnonPages: 281748 kB +Node 0 Mapped: 495936 kB +Node 0 Shmem: 2340 kB +Node 0 Slab: 1097040 kB +Node 0 SReclaimable: 445164 kB +Node 0 SUnreclaim: 651876 kB +Node 0 KernelStack: 20944 kB +Node 0 PageTables: 7896 kB +Node 0 NFS_Unstable: 0 kB +Node 0 Bounce: 0 kB +Node 0 WritebackTmp: 0 kB +Node 0 AnonHugePages: 38912 kB +Node 0 ShmemHugePages: 0 kB +Node 0 ShmemPmdMapped: 0 kB +Node 0 HugePages_Total: 0 +Node 0 HugePages_Free: 0 +Node 0 HugePages_Rsvd: 0 +Node 0 HugePages_Surp: 0` + numaMemInfoContentStr1 := `Node 1 MemTotal: 263432000 kB +Node 1 MemFree: 254391744 kB +Node 1 MemAvailable: 256703236 kB +Node 1 Buffers: 958096 kB +Node 1 Cached: 0 kB +Node 1 SwapCached: 0 kB +Node 1 Active: 2786012 kB +Node 1 Inactive: 2223752 kB +Node 1 Active(anon): 289488 kB +Node 1 Inactive(anon): 1300 kB +Node 1 Active(file): 2496524 kB +Node 1 Inactive(file): 2222452 kB +Node 1 Unevictable: 0 kB +Node 1 Mlocked: 0 kB +Node 1 SwapTotal: 0 kB +Node 1 SwapFree: 0 kB +Node 1 Dirty: 624 kB +Node 1 Writeback: 0 kB +Node 1 AnonPages: 281748 kB +Node 1 Mapped: 495936 kB +Node 1 Shmem: 2340 kB +Node 1 Slab: 1097040 kB +Node 1 SReclaimable: 445164 kB +Node 1 SUnreclaim: 651876 kB +Node 1 KernelStack: 20944 kB +Node 1 PageTables: 7896 kB +Node 1 NFS_Unstable: 0 kB +Node 1 Bounce: 0 kB +Node 1 WritebackTmp: 0 kB +Node 1 AnonHugePages: 38912 kB +Node 1 ShmemHugePages: 0 kB +Node 1 ShmemPmdMapped: 0 kB +Node 1 HugePages_Total: 0 +Node 1 HugePages_Free: 0 +Node 1 HugePages_Rsvd: 0 +Node 1 HugePages_Surp: 0` + numaMemInfoPath0 := system.GetNUMAMemInfoPath("node0") + helper.WriteFileContents(numaMemInfoPath0, numaMemInfoContentStr0) + numaMemInfoPath1 := system.GetNUMAMemInfoPath("node1") + helper.WriteFileContents(numaMemInfoPath1, numaMemInfoContentStr1) + + numaHugePage1GPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath0, "10") + numaHugePage2MPath0 := system.GetNUMAHugepagesNrPath("node0", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath0, "20") + + numaHugePage1GPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-1048576kB") + helper.WriteFileContents(numaHugePage1GPath1, "10") + numaHugePage2MPath1 := system.GetNUMAHugepagesNrPath("node1", "hugepages-2048kB") + helper.WriteFileContents(numaHugePage2MPath1, "20") + + testMemInfo0 := &MemInfo{ + MemTotal: 263432804, MemFree: 254391744, MemAvailable: 256703236, + Buffers: 958096, Cached: 0, SwapCached: 0, + Active: 2786012, Inactive: 2223752, ActiveAnon: 289488, + InactiveAnon: 1300, ActiveFile: 2496524, InactiveFile: 2222452, + Unevictable: 0, Mlocked: 0, SwapTotal: 0, + SwapFree: 0, Dirty: 624, Writeback: 0, + AnonPages: 281748, Mapped: 495936, Shmem: 2340, + Slab: 1097040, SReclaimable: 445164, SUnreclaim: 651876, + KernelStack: 20944, PageTables: 7896, NFS_Unstable: 0, + Bounce: 0, WritebackTmp: 0, AnonHugePages: 38912, + HugePages_Total: 0, HugePages_Free: 0, HugePages_Rsvd: 0, + HugePages_Surp: 0, + } + testMemInfo1 := &MemInfo{ + MemTotal: 263432000, MemFree: 254391744, MemAvailable: 256703236, + Buffers: 958096, Cached: 0, SwapCached: 0, + Active: 2786012, Inactive: 2223752, ActiveAnon: 289488, + InactiveAnon: 1300, ActiveFile: 2496524, InactiveFile: 2222452, + Unevictable: 0, Mlocked: 0, SwapTotal: 0, + SwapFree: 0, Dirty: 624, Writeback: 0, + AnonPages: 281748, Mapped: 495936, Shmem: 2340, + Slab: 1097040, SReclaimable: 445164, SUnreclaim: 651876, + KernelStack: 20944, PageTables: 7896, NFS_Unstable: 0, + Bounce: 0, WritebackTmp: 0, AnonHugePages: 38912, + HugePages_Total: 0, HugePages_Free: 0, HugePages_Rsvd: 0, + HugePages_Surp: 0, + } + + testEmptyHugePagesMap0 := map[uint64]*HugePagesInfo{ + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + } + + testHugePagesMap0 := map[uint64]*HugePagesInfo{ + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + } + + testHugePagesMap1 := map[uint64]*HugePagesInfo{ + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + } + + expectedWithoutHugepage := &NodeNUMAInfo{ + NUMAInfos: []NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + }, + }, + MemInfoMap: map[int32]*MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + } + + expectedEmptyHugepage := &NodeNUMAInfo{ + NUMAInfos: []NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + HugePages: testEmptyHugePagesMap0, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + HugePages: testEmptyHugePagesMap0, + }, + }, + MemInfoMap: map[int32]*MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + HugePagesMap: map[int32]map[uint64]*HugePagesInfo{ + 0: { + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + }, + 1: { + Hugepage1Gkbyte: { + NumPages: 0, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 0, + PageSize: Hugepage2Mkbyte, + }, + }, + }, + } + + expected := &NodeNUMAInfo{ + NUMAInfos: []NUMAInfo{ + { + NUMANodeID: 0, + MemInfo: testMemInfo0, + HugePages: testHugePagesMap0, + }, + { + NUMANodeID: 1, + MemInfo: testMemInfo1, + HugePages: testHugePagesMap1, + }, + }, + MemInfoMap: map[int32]*MemInfo{ + 0: testMemInfo0, + 1: testMemInfo1, + }, + HugePagesMap: map[int32]map[uint64]*HugePagesInfo{ + 0: { + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + }, + 1: { + Hugepage1Gkbyte: { + NumPages: 10, + PageSize: Hugepage1Gkbyte, + }, + Hugepage2Mkbyte: { + NumPages: 20, + PageSize: Hugepage2Mkbyte, + }, + }, + }, + } + + numaInfo, err := GetNodeNUMAInfo() + assert.NoError(t, err) + assert.Equal(t, expectedWithoutHugepage, numaInfo) + + numaInfo1, err := GetNodeNUMAInfo() + assert.NoError(t, err) + assert.Equal(t, expectedWithoutHugepage, numaInfo1) + + numaInfo2, err := GetNodeNUMAInfo() + assert.NoError(t, err) + assert.Equal(t, expectedWithoutHugepage, numaInfo2) + + numaInfoWithHugePage := GetAndMergeHugepageToNumaInfo(numaInfo) + assert.Equal(t, expected, numaInfoWithHugePage) + + // test partial failure + + numaMemInfoPath2 := system.GetNUMAMemInfoPath("node2") + helper.MkDirAll(filepath.Dir(numaMemInfoPath2)) + fmt.Printf("%v", numaInfo1) + numaInfoWithHugePageErr := GetAndMergeHugepageToNumaInfo(numaInfo1) + assert.Equal(t, expectedEmptyHugepage, numaInfoWithHugePageErr) + + // test path not exist + helper.Cleanup() + numaInfoWithHugePageErr2 := GetAndMergeHugepageToNumaInfo(numaInfo2) + assert.Equal(t, expectedEmptyHugepage, numaInfoWithHugePageErr2) +} diff --git a/pkg/koordlet/util/system/system_file.go b/pkg/koordlet/util/system/system_file.go index 5fa211fc1..3e47eab89 100644 --- a/pkg/koordlet/util/system/system_file.go +++ b/pkg/koordlet/util/system/system_file.go @@ -36,6 +36,8 @@ const ( SysctlSubDir = "sys" ProcCPUInfoName = "cpuinfo" KernelCmdlineFileName = "cmdline" + HugepageDir = "hugepages" + nrPath = "nr_hugepages" KernelSchedGroupIdentityEnable = "kernel/sched_group_identity_enabled" @@ -103,6 +105,14 @@ func GetNUMAMemInfoPath(numaNodeSubDir string) string { return filepath.Join(Conf.SysRootDir, SysNUMASubDir, numaNodeSubDir, ProcMemInfoName) } +func GetNUMAHugepagesDir(numaNodeSubDir string) string { + return filepath.Join(Conf.SysRootDir, SysNUMASubDir, numaNodeSubDir, HugepageDir) +} + +func GetNUMAHugepagesNrPath(numaNodeSubDir string, page string) string { + return filepath.Join(Conf.SysRootDir, SysNUMASubDir, numaNodeSubDir, HugepageDir, page, nrPath) +} + func GetCPUInfoPath() string { return filepath.Join(Conf.ProcRootDir, ProcCPUInfoName) }