[network metrics] instance network interface metrics

Overview
========

This is related to the ongoing work in [plumbing instance/guest
metrics](#742) through to an
oximeter producer in propolis.

Includes:
* An update to `NetworkInterfaceRequest` to include an interface_id, which is part of the request on the [sled instance in Omicron](oxidecomputer/omicron#6414).
* Within initialization, with concrete types, we create interface_ids mapping nic UUIDs => `minor` number of the device instance of viona.
  - The device instance number matches the Kstat we'll look for.
  - We track this mapping via a type alias `InterfaceIdentifiers`, which is passed through from Machine initialization to VM objects (and VM objects shared), the latter of which we can use as part of registering the oximeter producer.
* A new stats module for collecting network_interface metrics.
  - In `to_samples` we generate multiple network interface targets based on vnics per instance, appropriately updating the `interface_id`, which is tracked in `interface_ids`.
* Move kstat_types out of virtual machine-specific stats and use its mocking for other stats tests

Dependencies
------------

- [X] oxidecomputer/omicron#6414
- [X] https://code.illumos.org/c/illumos-gate/+/3630

Cargo.toml

@@ -81,8 +81,8 @@ oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 sled-agent-client = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 
 # Crucible
-crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "d037eeb9a56a8a62ff17266f340c011224d15146" }
-crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "d037eeb9a56a8a62ff17266f340c011224d15146" }
+crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
+crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" }
 
 # External dependencies
 anyhow = "1.0"
@@ -96,7 +96,7 @@ bitflags = "2.4"
 bitstruct = "0.1"
 bitvec = "1.0"
 byteorder = "1"
-bytes = "1.1"
+bytes = "1.7.1"
 camino = "1.1.6"
 cargo_metadata = "0.18.1"
 cc = "1.0.73"
@@ -164,3 +164,20 @@ tracing-subscriber = "0.3.14"
 usdt = { version = "0.5", default-features = false }
 uuid = "1.3.2"
 zerocopy = "0.7.34"
+
+
+#
+# It's common during development to use a local copy of various complex
+# dependencies.  If you want to use those, uncomment one of these blocks.
+#
+# [patch."https://github.com/oxidecomputer/omicron"]
+# internal-dns = { path = "../omicron/internal-dns" }
+# nexus-client = { path = "../omicron/clients/nexus-client" }
+# omicron-common = { path = "../omicron/common" }
+# omicron-zone-package = "0.9.0"
+# oximeter-instruments = { path = "../omicron/oximeter/instruments" }
+# oximeter-producer = { path = "../omicron/oximeter/producer" }
+# oximeter = { path = "../omicron/oximeter/oximeter" }
+# [patch."https://github.com/oxidecomputer/crucible"]
+# crucible = { path = "../crucible/upstairs" }
+# crucible-client-types = { path = "../crucible/crucible-client-types" }

bin/propolis-server/src/lib/initializer.rs

@@ -12,15 +12,16 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::serial::Serial;
 use crate::stats::{
-    create_kstat_sampler, track_vcpu_kstats, virtual_disk::VirtualDiskProducer,
-    virtual_machine::VirtualMachine,
+    create_kstat_sampler, track_network_interface_kstats, track_vcpu_kstats,
+    VirtualDiskProducer, VirtualMachine,
+};
+use crate::vm::{
+    BlockBackendMap, CrucibleBackendMap, DeviceMap, NetworkInterfaceIds,
 };
-use crate::vm::{BlockBackendMap, CrucibleBackendMap, DeviceMap};
 use anyhow::{Context, Result};
 use crucible_client_types::VolumeConstructionRequest;
 pub use nexus_client::Client as NexusClient;
 use oximeter::types::ProducerRegistry;
-
 use oximeter_instruments::kstat::KstatSampler;
 use propolis::block;
 use propolis::chardev::{self, BlockingSource, Source};
@@ -41,7 +42,7 @@ use propolis_api_types::instance_spec::{self, v0::InstanceSpecV0};
 use propolis_api_types::InstanceProperties;
 use slog::info;
 
-// Arbitrary ROM limit for now
+/// Arbitrary ROM limit for now
 const MAX_ROM_SIZE: usize = 0x20_0000;
 
 fn get_spec_guest_ram_limits(spec: &InstanceSpecV0) -> (usize, usize) {
@@ -628,10 +629,20 @@ impl<'a> MachineInitializer<'a> {
         Ok(())
     }
 
-    pub fn initialize_network_devices(
+    /// Initialize network devices, add them to the device map, and attach them
+    /// to the chipset.
+    ///
+    /// If a KstatSampler is provided, this function will also track network
+    /// interface statistics.
+    pub async fn initialize_network_devices(
         &mut self,
         chipset: &RegisteredChipset,
+        kstat_sampler: &Option<KstatSampler>,
     ) -> Result<(), Error> {
+        // Only create the vector if the kstat_sampler exists.
+        let mut interface_ids: Option<NetworkInterfaceIds> =
+            kstat_sampler.as_ref().map(|_| Vec::new());
+
         for (name, vnic_spec) in &self.spec.devices.network_devices {
             info!(self.log, "Creating vNIC {}", name);
             let instance_spec::v0::NetworkDeviceV0::VirtioNic(vnic_spec) =
@@ -680,8 +691,26 @@ impl<'a> MachineInitializer<'a> {
             )?;
             self.devices
                 .insert(format!("pci-virtio-viona-{}", bdf), viona.clone());
+
+            // Only push to interface_ids if kstat_sampler exists
+            if let Some(ref mut ids) = interface_ids {
+                ids.push((vnic_spec.interface_id, viona.instance_id()?));
+            }
+
             chipset.pci_attach(bdf, viona);
         }
+
+        kstat_sampler.as_ref().map(|sampler| {
+            interface_ids.map(|ids| {
+                track_network_interface_kstats(
+                    &self.log,
+                    sampler,
+                    self.properties,
+                    ids,
+                )
+            })
+        });
+
         Ok(())
     }
 
@@ -1088,6 +1117,9 @@ impl<'a> MachineInitializer<'a> {
         Ok(ramfb)
     }
 
+    /// Initialize virtual CPUs by first setting their capabilities, inserting
+    /// them into the device map, and then, if a kstat sampler is provided,
+    /// tracking their kstats.
     pub async fn initialize_cpus(
         &mut self,
         kstat_sampler: &Option<KstatSampler>,
@@ -1109,8 +1141,12 @@ impl<'a> MachineInitializer<'a> {
         let Some(registry) = &self.producer_registry else {
             return None;
         };
-        let sampler =
-            create_kstat_sampler(&self.log, u32::from(self.properties.vcpus))?;
+        let sampler = create_kstat_sampler(
+            &self.log,
+            u32::from(self.properties.vcpus),
+            self.spec.devices.network_devices.len() as u32,
+        )?;
+
         match registry.register_producer(sampler.clone()) {
             Ok(_) => Some(sampler),
             Err(e) => {

bin/propolis-server/src/lib/spec/api_request.rs

@@ -138,6 +138,7 @@ pub(super) fn parse_nic_from_request(
     let (device_name, backend_name) = super::pci_path_to_nic_names(pci_path);
     let device_spec = NetworkDeviceV0::VirtioNic(VirtioNic {
         backend_name: backend_name.clone(),
+        interface_id: nic.interface_id,
         pci_path,
     });
 
@@ -193,8 +194,11 @@ mod test {
 
     #[test]
     fn parsed_network_devices_point_to_backends() {
-        let req =
-            NetworkInterfaceRequest { name: "vnic".to_string(), slot: Slot(0) };
+        let req = NetworkInterfaceRequest {
+            name: "vnic".to_string(),
+            interface_id: uuid::Uuid::new_v4(),
+            slot: Slot(0),
+        };
 
         let parsed = parse_nic_from_request(&req).unwrap();
         let NetworkDeviceV0::VirtioNic(nic) = &parsed.device_spec;

bin/propolis-server/src/lib/spec/config_toml.rs

@@ -291,6 +291,9 @@ pub(super) fn parse_network_device_from_config(
 
     let device_spec = NetworkDeviceV0::VirtioNic(VirtioNic {
         backend_name: backend_name.clone(),
+        // We don't allow for configuration to specify the interface_id, so we
+        // generate a new one.
+        interface_id: uuid::Uuid::new_v4(),
         pci_path,
     });