forked from alibaba/kubeskoop
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request alibaba#265 from BSWANG/main
feat: support rdma resource metrics
- Loading branch information
Showing
6 changed files
with
261 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package rdma | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/alibaba/kubeskoop/pkg/exporter/probe" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/samber/lo" | ||
| ) | ||
|
|
||
| const ( | ||
| linkTypeERdma = "erdma" | ||
| ) | ||
|
|
||
| var ( | ||
| erdmaStatisticCounterEntries = map[string]string{ | ||
| "accept_failed_cnt": "The total number of failed connection accept attempts.", | ||
| "accept_success_cnt": "The total number of successful connection accept attempts.", | ||
| "accept_total_cnt": "The total number of connection accept attempts, successful or not.", | ||
| "cmdq_comp_cnt": "The total number of command queue completions processed.", | ||
| "cmdq_cq_armed_cnt": "The total number of command queue completion events that have been armed.", | ||
| "cmdq_eq_event_cnt": "The total number of command queue event queue events received.", | ||
| "cmdq_eq_notify_cnt": "The total number of command queue event queue notifications triggered.", | ||
| "cmdq_submitted_cnt": "The total number of command queue submissions.", | ||
| "connect_failed_cnt": "The total number of failed connection attempts.", | ||
| "connect_reset_cnt": "The total number of connection attempts that have been reset.", | ||
| "connect_success_cnt": "The total number of successful connection attempts.", | ||
| "connect_timeout_cnt": "The total number of connection attempts that timed out.", | ||
| "connect_total_cnt": "The total number of connection attempts, successful or not.", | ||
| "erdma_aeq_event_cnt": "The total number of ERDMA asynchronous event queue events received.", | ||
| "erdma_aeq_notify_cnt": "The total number of ERDMA asynchronous event queue notifications triggered.", | ||
| "hw_bps_limit_drop_cnt": "The total number of packets dropped due to hardware bandwidth limit.", | ||
| "hw_disable_drop_cnt": "The total number of packets dropped due to hardware being disabled.", | ||
| "hw_pps_limit_drop_cnt": "The total number of packets dropped due to hardware packets-per-second limit.", | ||
| "hw_rx_bps_limit_drop_cnt": "The total number of received packets dropped due to hardware receive bandwidth limit.", | ||
| "hw_rx_bytes_cnt": "The total number of bytes received by the hardware.", | ||
| "hw_rx_disable_drop_cnt": "The total number of received packets dropped due to receive hardware being disabled.", | ||
| "hw_rx_packets_cnt": "The total number of packets received by the hardware.", | ||
| "hw_rx_pps_limit_drop_cnt": "The total number of received packets dropped due to hardware receive packets-per-second limit.", | ||
| "hw_tx_bytes_cnt": "The total number of bytes transmitted by the hardware.", | ||
| "hw_tx_packets_cnt": "The total number of packets transmitted by the hardware.", | ||
| "hw_tx_reqs_cnt": "The total number of transmit requests processed by the hardware.", | ||
| "listen_create_cnt": "The total number of successfully created listen sockets.", | ||
| "listen_destroy_cnt": "The total number of destroyed listen sockets.", | ||
| "listen_failed_cnt": "The total number of failed attempts to create listen sockets.", | ||
| "listen_ipv6_cnt": "The total number of listen sockets created for IPv6 addresses.", | ||
| "listen_success_cnt": "The total number of successful listen operations.", | ||
| "reject_cnt": "The total number of received connection requests that were rejected.", | ||
| "reject_failed_cnt": "The total number of failed attempts to reject connection requests.", | ||
| "verbs_alloc_mr_cnt": "The total number of successful memory region allocations using verbs API.", | ||
| "verbs_alloc_mr_failed_cnt": "The total number of failed memory region allocation attempts using verbs API.", | ||
| "verbs_alloc_pd_cnt": "The total number of successful protection domain allocations using verbs API.", | ||
| "verbs_alloc_pd_failed_cnt": "The total number of failed protection domain allocation attempts using verbs API.", | ||
| "verbs_alloc_uctx_cnt": "The total number of successful user context allocations using verbs API.", | ||
| "verbs_alloc_uctx_failed_cnt": "The total number of failed user context allocation attempts using verbs API.", | ||
| "verbs_create_cq_cnt": "The total number of successful completion queue creations using verbs API.", | ||
| "verbs_create_cq_failed_cnt": "The total number of failed completion queue creation attempts using verbs API.", | ||
| "verbs_destroy_cq_failed_cnt": "The total number of failed completion queue deletion using verbs API.", | ||
| "verbs_create_qp_cnt": "The total number of successful queue pair creations using verbs API.", | ||
| "verbs_create_qp_failed_cnt": "The total number of failed queue pair creation attempts using verbs API.", | ||
| "verbs_destroy_qp_cnt": "The total number of failed queue pair deletion using verbs API.", | ||
| "verbs_dealloc_pd_cnt": "The total number of deallocated protection domains using verbs API.", | ||
| "verbs_dealloc_uctx_cnt": "The total number of deallocated user contexts using verbs API.", | ||
| "verbs_dereg_mr_cnt": "The total number of successful memory region deregistrations using verbs API.", | ||
| "verbs_dereg_mr_failed_cnt": "The total number of failed memory region deregistration attempts using verbs API.", | ||
| "verbs_destroy_cq_cnt": "The total number of destroyed completion queues using verbs API.", | ||
| "verbs_destroy_qp_failed_cnt": "The total number of failed attempts to destroy queue pairs (QPs) using verbs API.", | ||
| "verbs_get_dma_mr_cnt": "The total number of successful direct memory access (DMA) memory region acquisitions using verbs API.", | ||
| "verbs_get_dma_mr_failed_cnt": "The total number of failed attempts to acquire direct memory access (DMA) memory regions using verbs API.", | ||
| "verbs_reg_usr_mr_cnt": "The total number of user memory regions successfully registered with the verbs API.", | ||
| "verbs_reg_usr_mr_failed_cnt": "The total number of failed attempts to register user memory regions with the verbs API.", | ||
| } | ||
| erdmaMetrics = lo.Map(lo.Keys(erdmaStatisticCounterEntries), func(k string, _ int) probe.SingleMetricsOpts { | ||
| return probe.SingleMetricsOpts{ | ||
| Name: strings.Join([]string{linkTypeERdma, k}, "_"), | ||
| VariableLabels: rdmaDevPortLabels, | ||
| Help: erdmaStatisticCounterEntries[k], | ||
| ValueType: prometheus.CounterValue, | ||
| } | ||
| }) | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package rdma | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/alibaba/kubeskoop/pkg/exporter/probe" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/samber/lo" | ||
| ) | ||
|
|
||
| const ( | ||
| linkTypeMellanox = "mellanox_mlx5" | ||
| ) | ||
|
|
||
| var ( | ||
| mlx5 = map[string]string{ | ||
| "rx_write_requests": "The number of received WRITE requests for the associated QPs.", | ||
| "rx_read_requests": "The number of received READ requests for the associated QPs.", | ||
| "rx_atomic_requests": "The number of received ATOMIC request for the associated QPs.", | ||
| "out_of_buffer": "The number of drops occurred due to lack of WQE for the associated QPs.", | ||
| "out_of_sequence": "The number of out of sequence packets received.", | ||
| "duplicate_request": "Number of duplicate request packets.", | ||
| "rnr_nak_retry_err": "The number of received RNR NAK packets. The QP retry limit was not exceeded.", | ||
| "packet_seq_err": "The number of received NAK sequence error packets. The QP retry limit was not exceeded.", | ||
| "implied_nak_seq_err": "Number of time the requested decided an ACK with a PSN larger than the expected PSN for an RDMA read or response.", | ||
| "local_ack_timeout_err": "The number of times QP's ack timer expired for RC, XRC, DCT QPs at the sender side.", | ||
| "rx_dct_connect": "The number of received connection request for the associated DCTs.", | ||
| "resp_local_length_error": "The number of times responder detected local length errors.", | ||
| "resp_cqe_error": "The number of times responder detected CQEs completed with errors.", | ||
| "req_cqe_error": "The number of times requester detected CQEs completed with errors.", | ||
| "req_remote_invalid_request": "The number of times requester detected remote invalid request errors.", | ||
| "req_remote_access_errors": "The number of times requester detected remote access errors.", | ||
| "resp_remote_access_errors": "The number of times responder detected remote access errors.", | ||
| "resp_cqe_flush_error": "The number of times responder detected CQEs completed with flushed errors.", | ||
| "req_cqe_flush_error": "The number of times requester detected CQEs completed with flushed errors.", | ||
| "roce_adp_retrans": "The number of adaptive retransmissions for RoCE traffic", | ||
| "roce_adp_retrans_to": "The number of times RoCE traffic reached timeout due to adaptive retransmission", | ||
| "roce_slow_restart": "The number of times RoCE slow restart was used", | ||
| "roce_slow_restart_cnps": "The number of times RoCE slow restart generated CNP packets", | ||
| "roce_slow_restart_trans": "The number of times RoCE slow restart changed state to slow restart", | ||
| "rp_cnp_ignored": "The number of CNP packets received and ignored by the Reaction Point HCA.", | ||
| "rp_cnp_handled": "The number of CNP packets handled by the Reaction Point HCA to throttle the transmission rate.", | ||
| "np_ecn_marked_roce_packets": "The number of RoCEv2 packets received by the notification point which were marked for experiencing the congestion (ECN bits where '11' on the ingress RoCE traffic) .", | ||
| "np_cnp_sent": "The number of CNP packets sent by the Notification Point when it noticed congestion experienced in the RoCEv2 IP header (ECN bits).", | ||
| "rx_icrc_encapsulated": "The number of RoCE packets with ICRC errors.", | ||
| } | ||
| mlx5Metrics = lo.Map(lo.Keys(mlx5), func(k string, _ int) probe.SingleMetricsOpts { | ||
| return probe.SingleMetricsOpts{ | ||
| Name: strings.Join([]string{linkTypeMellanox, k}, "_"), | ||
| VariableLabels: rdmaDevPortLabels, | ||
| Help: mlx5[k], | ||
| ValueType: prometheus.CounterValue, | ||
| } | ||
| }) | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package rdma | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| "github.com/alibaba/kubeskoop/pkg/exporter/nettop" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/samber/lo" | ||
| log "github.com/sirupsen/logrus" | ||
| "github.com/vishvananda/netlink" | ||
|
|
||
| "github.com/alibaba/kubeskoop/pkg/exporter/probe" | ||
| ) | ||
|
|
||
| const ( | ||
| probeName = "rdma" | ||
| linkTypeUnknown = "unknown" | ||
| ) | ||
|
|
||
| var ( | ||
| resourceSummaryEntries = []string{"cm_id", "cq", "ctx", "mr", "pd", "qp"} | ||
| rdmaDevLabels = []string{"device", "type"} | ||
| rdmaDevPortLabels = append(rdmaDevLabels, "port") | ||
| ) | ||
|
|
||
| func init() { | ||
| probe.MustRegisterMetricsProbe(probeName, metricsProbeCreator) | ||
| } | ||
|
|
||
| func metricsProbeCreator() (probe.MetricsProbe, error) { | ||
| p := &metricsProbe{} | ||
|
|
||
| opts := probe.BatchMetricsOpts{ | ||
| Namespace: probe.MetricsNamespace, | ||
| Subsystem: probeName, | ||
| VariableLabels: probe.StandardMetricsLabels, | ||
| SingleMetricsOpts: lo.Map(resourceSummaryEntries, func(entry string, _ int) probe.SingleMetricsOpts { | ||
| return probe.SingleMetricsOpts{Name: entry, VariableLabels: rdmaDevLabels, Help: fmt.Sprintf("rdma resource summary %s", entry), ValueType: prometheus.GaugeValue} | ||
| }), | ||
| } | ||
| opts.SingleMetricsOpts = append(opts.SingleMetricsOpts, mlx5Metrics...) | ||
| opts.SingleMetricsOpts = append(opts.SingleMetricsOpts, erdmaMetrics...) | ||
| batchMetrics := probe.NewBatchMetrics(opts, p.collectOnce) | ||
| return probe.NewMetricsProbe(probeName, p, batchMetrics), nil | ||
| } | ||
|
|
||
| type metricsProbe struct { | ||
| } | ||
|
|
||
| func (p *metricsProbe) Start(_ context.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (p *metricsProbe) Stop(_ context.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (p *metricsProbe) collectOnce(emit probe.Emit) error { | ||
| // rdma only collect host network | ||
| entity, err := nettop.GetHostNetworkEntity() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| rdmaRes, err := netlink.RdmaResourceList() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if len(rdmaRes) == 0 { | ||
| return nil | ||
| } | ||
| standardLabelValues := probe.BuildStandardMetricsLabelValues(entity) | ||
| for _, res := range rdmaRes { | ||
| link, err := netlink.RdmaLinkByName(res.Name) | ||
| if err != nil { | ||
| log.Errorf("failed get rdma link %v, error: %v", res.Name, err) | ||
| continue | ||
| } | ||
| linkType := rdmaLinkType(link) | ||
| deviceLabelValues := append(standardLabelValues, res.Name, linkType) | ||
| for resKey, resVal := range res.RdmaResourceSummaryEntries { | ||
| emit(resKey, deviceLabelValues, float64(resVal)) | ||
| } | ||
| if linkType == "unknown" { | ||
| continue | ||
| } | ||
| linkStatistics, err := netlink.RdmaStatistic(link) | ||
| if err != nil { | ||
| log.Errorf("failed get rdma statistics %v, error: %v", res.Name, err) | ||
| continue | ||
| } | ||
|
|
||
| for _, port := range linkStatistics.RdmaPortStatistics { | ||
| devicePortLabelValues := append(deviceLabelValues, strconv.FormatUint(uint64(port.PortIndex), 10)) | ||
| for statKey, statVal := range port.Statistics { | ||
| emit(strings.Join([]string{linkType, statKey}, "_"), devicePortLabelValues, float64(statVal)) | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func rdmaLinkType(link *netlink.RdmaLink) string { | ||
| if link == nil { | ||
| return linkTypeUnknown | ||
| } | ||
| switch strings.Split(link.Attrs.Name, "_")[0] { | ||
| case linkTypeMellanox: | ||
| return linkTypeMellanox | ||
| case linkTypeERdma: | ||
| return linkTypeERdma | ||
| default: | ||
| return linkTypeUnknown | ||
| } | ||
| } |