From 5768f71bf94546f32c8f587cd6d8be3c845735c1 Mon Sep 17 00:00:00 2001
From: Kenny Ho <y2kenny@gmail.com>
Date: Thu, 11 Feb 2021 16:57:26 -0500
Subject: [PATCH] Add topology support
The Topology Manager is a Kubelet component that allows resources to be
co-ordinated in a Topology aligned manner. In order to do this, the
Device Plugin API was extended to include a TopologyInfo struct since
Kubernetes v1.17.
https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#device-plugin-integration-with-the-topology-manager
This commit add topology support for AMD GPU using hwloc.
---
Dockerfile | 2 +
README.md | 3 +-
cmd/k8s-device-plugin/main.go | 63 +++++++++++++++++++----
internal/pkg/hwloc/hwloc.go | 97 +++++++++++++++++++++++++++++++++++
4 files changed, 152 insertions(+), 13 deletions(-)
create mode 100644 internal/pkg/hwloc/hwloc.go
diff --git a/Dockerfile b/Dockerfile
index 52fc863a..e3400eae 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,6 +13,7 @@
# limitations under the License.
FROM golang:1.15.7-alpine3.13
RUN apk --no-cache add git pkgconfig build-base libdrm-dev
+RUN apk --no-cache add hwloc-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
RUN mkdir -p /go/src/github.com/RadeonOpenCompute/k8s-device-plugin
ADD . /go/src/github.com/RadeonOpenCompute/k8s-device-plugin
RUN go install \
@@ -22,6 +23,7 @@ RUN go install \
FROM alpine:3.13
MAINTAINER Kenny Ho <Kenny.Ho@amd.com>
RUN apk --no-cache add ca-certificates libdrm
+RUN apk --no-cache add hwloc --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
WORKDIR /root/
COPY --from=0 /go/bin/k8s-device-plugin .
CMD ["./k8s-device-plugin", "-logtostderr=true", "-stderrthreshold=INFO", "-v=5"]
diff --git a/README.md b/README.md
index 2921a3e0..b45cfe3c 100644
--- a/README.md
+++ b/README.md
@@ -16,8 +16,7 @@ More information about [RadeonOpenCompute (ROCm)][rocm]
## Limitations
-* This is an early prototype/alpha.
-* This plugin currently support device plugin API v1 only. This means it will only work with k8s v1.16+.
+* This plugin targets Kubernetes v1.18+.
## Deployment
The device plugin needs to be run on all the nodes that are equipped with AMD GPU. The simplist way of doing so is to create a Kubernetes [DaemonSet][ds], which run a copy of a pod on all (or some) Nodes in the cluster. We have a pre-built Docker image on [DockerHub][dhk8samdgpudp] that you can use for with your DaemonSet. This repository also have a pre-defined yaml file named `k8s-ds-amdgpu-dp.yaml`. You can create a DaemonSet in your Kubernetes cluster by running this command:
diff --git a/cmd/k8s-device-plugin/main.go b/cmd/k8s-device-plugin/main.go
index 076d4870..f2a014ef 100644
--- a/cmd/k8s-device-plugin/main.go
+++ b/cmd/k8s-device-plugin/main.go
@@ -28,6 +28,7 @@ import (
"time"
"github.com/RadeonOpenCompute/k8s-device-plugin/internal/pkg/amdgpu"
+ "github.com/RadeonOpenCompute/k8s-device-plugin/internal/pkg/hwloc"
"github.com/golang/glog"
"github.com/kubevirt/device-plugin-manager/pkg/dpm"
"golang.org/x/net/context"
@@ -129,14 +130,46 @@ func (p *Plugin) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListA
devs := make([]*pluginapi.Device, len(p.AMDGPUs))
- i := 0
- for id := range p.AMDGPUs {
- devs[i] = &pluginapi.Device{
- ID: id,
- Health: pluginapi.Healthy,
+ // limit scope for hwloc
+ func() {
+ var hw hwloc.Hwloc
+ hw.Init()
+ defer hw.Destroy()
+
+ i := 0
+ for id := range p.AMDGPUs {
+ dev := &pluginapi.Device{
+ ID: id,
+ Health: pluginapi.Healthy,
+ }
+ devs[i] = dev
+ i++
+
+ numas, err := hw.GetNUMANodes(id)
+ glog.Infof("Watching GPU with bus ID: %s NUMA Node: %+v", id, numas)
+ if err != nil {
+ glog.Error(err)
+ continue
+ }
+
+ if len(numas) == 0 {
+ glog.Errorf("No NUMA for GPU ID: %s", id)
+ continue
+ }
+
+ numaNodes := make([]*pluginapi.NUMANode, len(numas))
+ for j, v := range numas {
+ numaNodes[j] = &pluginapi.NUMANode{
+ ID: int64(v),
+ }
+ }
+
+ dev.Topology = &pluginapi.TopologyInfo{
+ Nodes: numaNodes,
+ }
}
- i++
- }
+ }()
+
s.Send(&pluginapi.ListAndWatchResponse{Devices: devs})
for {
@@ -241,9 +274,16 @@ func (l *Lister) NewPlugin(resourceLastName string) dpm.PluginInterface {
var gitDescribe string
func main() {
+ versions := [...]string{
+ "AMD GPU device plugin for Kubernetes",
+ fmt.Sprintf("%s version %s", os.Args[0], gitDescribe),
+ fmt.Sprintf("%s", hwloc.GetVersions()),
+ }
+
flag.Usage = func() {
- fmt.Fprintf(os.Stderr, "AMD GPU device plugin for Kubernetes\n")
- fmt.Fprintf(os.Stderr, "%s version %s\n", os.Args[0], gitDescribe)
+ for _, v := range versions {
+ fmt.Fprintf(os.Stderr, "%s\n", v)
+ }
fmt.Fprintln(os.Stderr, "Usage:")
flag.PrintDefaults()
}
@@ -252,8 +292,9 @@ func main() {
// this is also needed to enable glog usage in dpm
flag.Parse()
- glog.Infof("AMD GPU device plugin for Kubernetes")
- glog.Infof("%s version %s\n", os.Args[0], gitDescribe)
+ for _, v := range versions {
+ glog.Infof("%s", v)
+ }
l := Lister{
ResUpdateChan: make(chan dpm.PluginNameList),
diff --git a/internal/pkg/hwloc/hwloc.go b/internal/pkg/hwloc/hwloc.go
new file mode 100644
index 00000000..f12c989f
--- /dev/null
+++ b/internal/pkg/hwloc/hwloc.go
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2021 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+**/
+
+// Package hwloc is a collection of utility functions to get NUMA membership
+// of AMD GPU via the hwloc library
+package hwloc
+
+// #cgo pkg-config: hwloc
+// #include <stdint.h>
+// #include <hwloc.h>
+import "C"
+import (
+ "fmt"
+ "unsafe"
+)
+
+func GetVersions() string {
+ return fmt.Sprintf("hwloc: _VERSION: %s, _API_VERSION: %#08x, _COMPONENT_ABI: %d, Runtime: %#08x",
+ C.HWLOC_VERSION,
+ C.HWLOC_API_VERSION,
+ C.HWLOC_COMPONENT_ABI,
+ uint(C.hwloc_get_api_version()))
+}
+
+type Hwloc struct {
+ topology C.hwloc_topology_t
+}
+
+func (h *Hwloc) Init() error {
+ rc := C.hwloc_topology_init(&h.topology)
+ if rc != 0 {
+ return fmt.Errorf("Problem initializing hwloc topology rc: %d", rc)
+ }
+
+ rc = C.hwloc_topology_set_type_filter(h.topology,
+ C.HWLOC_OBJ_PCI_DEVICE,
+ C.HWLOC_TYPE_FILTER_KEEP_IMPORTANT)
+ if rc != 0 {
+ C.hwloc_topology_destroy(h.topology)
+ return fmt.Errorf("Problem setting type filter rc: %d", rc)
+ }
+
+ rc = C.hwloc_topology_load(h.topology)
+ if rc != 0 {
+ C.hwloc_topology_destroy(h.topology)
+ return fmt.Errorf("Problem loading topology rc: %d", rc)
+ }
+
+ return nil
+}
+
+func (h *Hwloc) Destroy() {
+ C.hwloc_topology_destroy(h.topology)
+}
+
+func (h *Hwloc) GetNUMANodes(busid string) ([]uint64, error) {
+ var gpu C.hwloc_obj_t
+ var ancestor C.hwloc_obj_t
+
+ busidstr := C.CString(busid)
+ defer C.free(unsafe.Pointer(busidstr))
+
+ gpu = C.hwloc_get_pcidev_by_busidstring(h.topology, busidstr)
+ if gpu == nil {
+ return []uint64{},
+ fmt.Errorf("Fail to find GPU with bus ID: %s", busid)
+ }
+ ancestor = C.hwloc_get_non_io_ancestor_obj(h.topology, gpu)
+
+ if ancestor == nil || ancestor.memory_arity <= 0 {
+ return []uint64{},
+ fmt.Errorf("No NUMA node found with bus ID: %s", busid)
+ }
+
+ var results []uint64
+ nn := ancestor.memory_first_child
+
+ for nn != nil {
+ results = append(results, uint64(nn.logical_index))
+ nn = nn.next_sibling
+ }
+
+ return results, nil
+}
--
GitLab