From 1729a42048409094933c8d5ed8d836b2fceff878 Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Fri, 29 Apr 2022 17:36:45 +0200 Subject: [PATCH] sriov: add support as special PCI devices Add support to report SRIOV devices. Differently from GPU devices, we model SRIOV devices as special PCI devices, extending the `pci` package instead of introducing a new top-level package. This design emerged during the review of a previous proposal: https://github.com/jaypipes/ghw/pull/230/commits/9058f61f48a3c801e7817dd10905fc11e57c298c#r755312597 SRIOV devices are either Physical Functions or Virtual functions. The preferred representation for ghw is Physical Functions, whose dependent devices will be Virtual Functions; however, for the sake of practicality, the API also exposes soft references to Virtual Functions, so consumers of the API can access them directly and not navigating the parent devices. This patch also adds support in `ghwc`, to report the sriov information, and in the `snapshot` package, to make sure to capture all the files in sysfs that ghw cares about. Last but not least, lacking access to suitable non-linux systems, support is provided only on linux OS, even though the API tries hard not to be linux-specific. Resolves: https://github.com/jaypipes/ghw/issues/92 Signed-off-by: Francesco Romani --- README.md | 33 +++++++++ cmd/ghwc/commands/sriov.go | 35 +++++++++ pkg/pci/function.go | 109 ++++++++++++++++++++++++++++ pkg/pci/function_linux.go | 87 ++++++++++++++++++++++ pkg/pci/function_stub.go | 13 ++++ pkg/pci/pci.go | 2 + pkg/pci/pci_linux.go | 1 + pkg/pci/pci_stub.go | 2 +- pkg/snapshot/clonetree.go | 15 +++- pkg/snapshot/clonetree_pci_linux.go | 32 ++++++++ 10 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 cmd/ghwc/commands/sriov.go create mode 100644 pkg/pci/function.go create mode 100644 pkg/pci/function_linux.go create mode 100644 pkg/pci/function_stub.go diff --git a/README.md b/README.md index 498a0f78..cabe52f5 100644 --- a/README.md +++ b/README.md @@ -1123,6 +1123,39 @@ information `ghw.TopologyNode` struct if you'd like to dig deeper into the NUMA/topology subsystem +### SRIOV + +SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + pci, err := ghw.PCI() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info through PCI: %v", err) + } + + fmt.Printf("%v\n", pci) + + for _, dev := range pci.Functions { + fmt.Printf(" %v\n", dev) + } +} +``` + +ghw discovers the SRIOV devices scanning PCI devices. Thus, you need to make sure to have scanned the PCI devices before +to query for SRIOV devices (aka "Functions", "PCI Functions"). +Virtual Functions (VFs) are handled by Physical Functions (PFs). +Virtual Functions are available both as entries in the `pci.Functions` slice and as properties of their parent Physical Functions. +Both references are aliases to the same object. + ### Chassis The host's chassis information is accessible with the `ghw.Chassis()` function. This diff --git a/cmd/ghwc/commands/sriov.go b/cmd/ghwc/commands/sriov.go new file mode 100644 index 00000000..c403d5bc --- /dev/null +++ b/cmd/ghwc/commands/sriov.go @@ -0,0 +1,35 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package commands + +import ( + "github.com/jaypipes/ghw" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// sriovCmd represents the install command +var sriovCmd = &cobra.Command{ + Use: "sriov", + Short: "Show Single Root I/O Virtualization device information for the host system", + RunE: showSRIOV, +} + +// showSRIOV show SRIOV information for the host system. +func showSRIOV(cmd *cobra.Command, args []string) error { + info, err := ghw.PCI() + if err != nil { + return errors.Wrap(err, "error getting SRIOV info through PCI") + } + + printInfo(info.DescribeFunctions()) + return nil +} + +func init() { + rootCmd.AddCommand(sriovCmd) +} diff --git a/pkg/pci/function.go b/pkg/pci/function.go new file mode 100644 index 00000000..ac08e463 --- /dev/null +++ b/pkg/pci/function.go @@ -0,0 +1,109 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "encoding/json" + "fmt" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/marshal" +) + +// Function describes an SR-IOV physical or virtual function. Physical functions +// will have no Parent Function struct pointer and will have one or more Function +// structs in the Functions field. +type Function struct { + Device // All Functions are PCI Devices + // Parent contains a pointer to the parent physical function. + // Will be empty when this is a physical function + Parent *Function `json:"parent,omitempty"` + // MaxVFs contains the maximum number of supported virtual + // functions for this physical function + MaxVFs int `json:"max_vfs"` + // Functions contains the physical function's virtual functions + Functions []*Function `json:"functions"` +} + +// IsPhysical returns true if the PCIe function is a physical function, false +// if it is a virtual function +func (f *Function) IsPhysical() bool { + return f.Parent == nil +} + +type fnMarshallable struct { + Driver string `json:"driver"` + Address string `json:"address"` + Vendor devIdent `json:"vendor"` + Product devIdent `json:"product"` + Type string `json:"type"` + ParentAddress string `json:"parent_address,omitempty"` + MaxVFs int `json:"virtual_functions_max,omitempty"` + Functions []string `json:"virtual_functions,omitempty"` +} + +func (f *Function) MarshalJSON() ([]byte, error) { + fm := fnMarshallable{ + Driver: f.Driver, + Address: f.Address, + Vendor: devIdent{ + ID: f.Vendor.ID, + Name: f.Vendor.Name, + }, + Product: devIdent{ + ID: f.Product.ID, + Name: f.Product.Name, + }, + MaxVFs: f.MaxVFs, + Type: "virtual", + } + if f.IsPhysical() { + fm.Type = "physical" + } + if f.Parent != nil { + fm.ParentAddress = f.Parent.Address + } + var fns []string + for _, fn := range f.Functions { + fns = append(fns, fn.Address) + } + fm.Functions = fns + return json.Marshal(fm) +} + +type FunctionsPrinter struct { + ctx *context.Context + Functions []*Function `json:"functions"` +} + +func (info *Info) DescribeFunctions() *FunctionsPrinter { + // this is to avoid ugly output like `{"functions":null}` + functions := info.Functions + if functions == nil { + functions = []*Function{} + } + return &FunctionsPrinter{ + ctx: info.ctx, + Functions: functions, + } +} + +func (fp *FunctionsPrinter) String() string { + return fmt.Sprintf("SRIOV (%d devices)", len(fp.Functions)) +} + +// YAMLString returns a string with the SRIOV information formatted as YAML +// under a top-level "functions:" key +func (fp *FunctionsPrinter) YAMLString() string { + return marshal.SafeYAML(fp.ctx, fp) +} + +// JSONString returns a string with the PCI information formatted as JSON +// under a top-level "pci:" key +func (fp *FunctionsPrinter) JSONString(indent bool) string { + return marshal.SafeJSON(fp.ctx, fp, indent) +} diff --git a/pkg/pci/function_linux.go b/pkg/pci/function_linux.go new file mode 100644 index 00000000..ea2b1131 --- /dev/null +++ b/pkg/pci/function_linux.go @@ -0,0 +1,87 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/util" +) + +func (info *Info) ListFunctions() []*Function { + var fns []*Function + for _, dev := range info.Devices { + pf := info.physicalFunctionFromDevice(dev) + if pf == nil { + // not a physical function, nothing to do + continue + } + fns = append(fns, pf) + fns = append(fns, pf.Functions...) + } + return fns +} + +func (info *Info) physicalFunctionFromDevice(dev *Device) *Function { + paths := linuxpath.New(info.ctx) + devPath := filepath.Join(paths.SysBusPciDevices, dev.Address) + + buf, err := ioutil.ReadFile(filepath.Join(devPath, "sriov_totalvfs")) + if err != nil { + // is not a physfn. Since we will fill virtfn from physfn, we can give up now + return nil + } + + maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf))) + if err != nil { + info.ctx.Warn("error reading sriov_totalvfn for %q: %v", devPath, err) + return nil + } + + fn := &Function{ + Device: *dev, + MaxVFs: maxVFs, + } + fn.Functions = virtualFunctionsFromPhysicalFunction(info, fn, devPath) + return fn +} + +func virtualFunctionsFromPhysicalFunction(info *Info, parentFn *Function, parentPath string) []*Function { + numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs")) + if numVfs == -1 { + return nil + } + + var vfs []*Function + for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ { + virtFn := fmt.Sprintf("virtfn%d", vfnIdx) + vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn)) + if err != nil { + info.ctx.Warn("error reading backing device for virtfn %q physfn %q: %v", virtFn, parentPath, err) + return nil + } + + vfnAddr := filepath.Base(vfnDest) + vfnDev := info.GetDevice(vfnAddr) + if vfnDev == nil { + info.ctx.Warn("error finding the PCI device for virtfn %s physfn %s", vfnAddr, parentFn.Address) + return nil + } + + // functions must be ordered by their index + vfs = append(vfs, &Function{ + Device: *vfnDev, + Parent: parentFn, + }) + } + return vfs +} diff --git a/pkg/pci/function_stub.go b/pkg/pci/function_stub.go new file mode 100644 index 00000000..b00997c0 --- /dev/null +++ b/pkg/pci/function_stub.go @@ -0,0 +1,13 @@ +//go:build !linux +// +build !linux + +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +func (info *Info) ListFunctions() []*Function { + return nil +} diff --git a/pkg/pci/pci.go b/pkg/pci/pci.go index 86cc7b25..18cbd4c4 100644 --- a/pkg/pci/pci.go +++ b/pkg/pci/pci.go @@ -127,6 +127,8 @@ type Info struct { ctx *context.Context // All PCI devices on the host system Devices []*Device + // All SRIOV devices on the host system (subset of Devices) + Functions []*Function `json:"-"` // hash of class ID -> class information // DEPRECATED. Will be removed in v1.0. Please use // github.com/jaypipes/pcidb to explore PCIDB information diff --git a/pkg/pci/pci_linux.go b/pkg/pci/pci_linux.go index 5fd6f0fa..ea4097d3 100644 --- a/pkg/pci/pci_linux.go +++ b/pkg/pci/pci_linux.go @@ -48,6 +48,7 @@ func (i *Info) load() error { i.Vendors = db.Vendors i.Products = db.Products i.Devices = i.ListDevices() + i.Functions = i.ListFunctions() return nil } diff --git a/pkg/pci/pci_stub.go b/pkg/pci/pci_stub.go index 9ebb396d..9e8b345b 100644 --- a/pkg/pci/pci_stub.go +++ b/pkg/pci/pci_stub.go @@ -15,7 +15,7 @@ import ( ) func (i *Info) load() error { - return errors.New("pciFillInfo not implemented on " + runtime.GOOS) + return errors.New("pci load() not implemented on " + runtime.GOOS) } // GetDevice returns a pointer to a Device struct that describes the PCI diff --git a/pkg/snapshot/clonetree.go b/pkg/snapshot/clonetree.go index 519a874d..587c6b03 100644 --- a/pkg/snapshot/clonetree.go +++ b/pkg/snapshot/clonetree.go @@ -100,7 +100,7 @@ func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) er if opts == nil { opts = &CopyFileOptions{ IsSymlinkFn: isSymlink, - ShouldCreateDirFn: isDriversDir, + ShouldCreateDirFn: shouldCreateDir, } } for _, fileSpec := range fileSpecs { @@ -157,6 +157,13 @@ func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) err return nil } +func shouldCreateDir(path string, fi os.FileInfo) bool { + if isDeviceNetworkDir(path, fi) { + return true + } + return isDriversDir(path, fi) +} + func isSymlink(path string, fi os.FileInfo) bool { return fi.Mode()&os.ModeSymlink != 0 } @@ -165,6 +172,12 @@ func isDriversDir(path string, fi os.FileInfo) bool { return strings.Contains(path, "drivers") } +func isDeviceNetworkDir(path string, fi os.FileInfo) bool { + parentDir := filepath.Base(filepath.Dir(path)) + // TODO: the "HasPrefix" check is brutal, but should work on linux + return parentDir == "net" && strings.HasPrefix(path, "/sys/devices") +} + func copyLink(path, targetPath string) error { target, err := os.Readlink(path) if err != nil { diff --git a/pkg/snapshot/clonetree_pci_linux.go b/pkg/snapshot/clonetree_pci_linux.go index dbc3fc83..e39e8c09 100644 --- a/pkg/snapshot/clonetree_pci_linux.go +++ b/pkg/snapshot/clonetree_pci_linux.go @@ -70,6 +70,19 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { "revision", "vendor", } + + perDevEntriesOpt := []string{ + "driver", + "net/*", + "physfn", + "sriov_*", + "virtfn*", + } + + ignoreSet := map[string]bool{ + "sriov_vf_msix_count": true, // linux >= 5.14, write-only + } + entries, err := ioutil.ReadDir(root) if err != nil { return []string{}, []string{} @@ -96,6 +109,25 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) } + for _, perNetEntryOpt := range perDevEntriesOpt { + netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt) + + items, err := filepath.Glob(netEntryOptPath) + if err != nil { + // TODO: we skip silently because we don't have + // a ctx handy, so we can't do ctx.Warn :\ + continue + } + + for _, item := range items { + globbedEntry := filepath.Base(item) + if _, ok := ignoreSet[globbedEntry]; ok { + continue + } + fileSpecs = append(fileSpecs, item) + } + } + if isPCIBridge(entryPath) { trace("adding new PCI root %q\n", entryName) pciRoots = append(pciRoots, pciEntry)