PHD: support guest-initiated reboot (#785)

While in many cases a procedure like "`reboot` and wait for login" is
sufficient to reboot a guest, it is not sufficient for *all* cases. For
Windows, the command is spelled `shutdown /r` instead. And on Debian (at
least 11), `reboot` will immediately take the guest down instead of
printing a new shell prompt first.

So, have guest OS adapters provide commands that impel a guest to
reboot, and wait for that reboot to occur (rather than the naive initial
approach of reboot, wait for shell prompt, wait for login prompt,
proceed).

Fixes #783.

phd-tests/framework/src/guest_os/alpine.rs

@@ -31,4 +31,12 @@ impl GuestOs for Alpine {
             crate::serial::BufferKind::Raw,
         )
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        // For Alpine guests we've looked at, `reboot` kicks off OpenRC behavior
+        // to reboot the system. We *could* wait for a new shell prompt at this
+        // point, but it's more reliable to wait for a guest to have fully
+        // rebooted and log back in.
+        self.shell_command_sequence("reboot")
+    }
 }

phd-tests/framework/src/guest_os/debian11_nocloud.rs

@@ -24,4 +24,16 @@ impl GuestOs for Debian11NoCloud {
     fn read_only_fs(&self) -> bool {
         false
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        // On Debian 11, `reboot` does not seem to be the same wrapper for
+        // `systemctl reboot` as it is on more recent Ubuntu. Whatever it *is*,
+        // it does its job before a new prompt line is printed, so we can only
+        // wait to see a new login sequence.
+        //
+        // While `systemctl reboot` does exist here, and is mechanically more
+        // like Ubuntu's `reboot`, just using `reboot` on Debian gets the job
+        // done and keeps our instructions consistent across Linuxes.
+        self.shell_command_sequence("reboot")
+    }
 }

phd-tests/framework/src/guest_os/mod.rs

@@ -79,6 +79,12 @@ pub(super) trait GuestOs: Send + Sync {
             crate::serial::BufferKind::Raw,
         )
     }
+
+    /// Returns the sequence of serial console operations a test VM must perform
+    /// in order to perform a graceful (e.g. guest-initiated and expected)
+    /// reboot. PHD's expectation following these commands will be to wait for
+    /// the guest's login sequence.
+    fn graceful_reboot(&self) -> CommandSequence;
 }
 
 #[allow(dead_code)]

phd-tests/framework/src/guest_os/ubuntu22_04.rs

@@ -27,4 +27,13 @@ impl GuestOs for Ubuntu2204 {
     fn read_only_fs(&self) -> bool {
         false
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        // Ubuntu `reboot` seems to be mechanically similar to Alpine `reboot`,
+        // except mediated by SystemD rather than OpenRC. We'll get a new shell
+        // prompt, and then the system reboots shortly after. Just issuing
+        // `reboot` and waiting for a login prompt is the lowest common
+        // denominator across Linuxes.
+        self.shell_command_sequence("reboot")
+    }
 }

phd-tests/framework/src/guest_os/windows_server_2016.rs

@@ -40,4 +40,8 @@ impl GuestOs for WindowsServer2016 {
             crate::serial::BufferKind::Vt80x24,
         )
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        self.shell_command_sequence("shutdown /r /t 0 /d p:0:0")
+    }
 }

phd-tests/framework/src/guest_os/windows_server_2019.rs

@@ -36,4 +36,8 @@ impl GuestOs for WindowsServer2019 {
             crate::serial::BufferKind::Vt80x24,
         )
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        self.shell_command_sequence("shutdown /r /t 0 /d p:0:0")
+    }
 }

phd-tests/framework/src/guest_os/windows_server_2022.rs

@@ -24,4 +24,8 @@ impl GuestOs for WindowsServer2022 {
     fn read_only_fs(&self) -> bool {
         false
     }
+
+    fn graceful_reboot(&self) -> CommandSequence {
+        self.shell_command_sequence("shutdown /r /t 0 /d p:0:0")
+    }
 }

phd-tests/framework/src/test_vm/mod.rs

@@ -8,7 +8,9 @@
 use std::{fmt::Debug, io::Write, sync::Arc, time::Duration};
 
 use crate::{
-    guest_os::{self, CommandSequenceEntry, GuestOs, GuestOsKind},
+    guest_os::{
+        self, CommandSequence, CommandSequenceEntry, GuestOs, GuestOsKind,
+    },
     serial::{BufferKind, SerialConsole},
     test_vm::{
         environment::Environment, server::ServerProcessParameters, spec::VmSpec,
@@ -872,6 +874,38 @@ impl TestVm {
         // type (which affects e.g. affects how it displays multi-line commands)
         // and serial console buffering discipline.
         let command_sequence = self.guest_os.shell_command_sequence(cmd);
+        self.run_command_sequence(command_sequence).await?;
+
+        // `shell_command_sequence` promises that the generated command sequence
+        // clears buffer of everything up to and including the input command
+        // before actually issuing the final '\n' that issues the command.
+        // This ensures that the buffer contents returned by this call contain
+        // only the command's output.
+        let out = self
+            .wait_for_serial_output(
+                self.guest_os.get_shell_prompt(),
+                Duration::from_secs(300),
+            )
+            .await?;
+
+        // Trim any leading newlines inserted when the command was issued and
+        // any trailing whitespace that isn't actually part of the command
+        // output. Any other embedded whitespace is the caller's problem.
+        Ok(out.trim().to_string())
+    }
+
+    pub async fn graceful_reboot(&self) -> Result<()> {
+        self.run_command_sequence(self.guest_os.graceful_reboot()).await?;
+        self.wait_to_boot().await
+    }
+
+    /// Run a [`CommandSequence`] in the context of a booted and logged-in
+    /// guest. The guest is expected to be at a shell prompt when this sequence
+    /// is begun.
+    async fn run_command_sequence(
+        &self,
+        command_sequence: CommandSequence<'_>,
+    ) -> Result<()> {
         for step in command_sequence.0 {
             match step {
                 CommandSequenceEntry::WaitFor(s) => {
@@ -896,22 +930,7 @@ impl TestVm {
             }
         }
 
-        // `shell_command_sequence` promises that the generated command sequence
-        // clears buffer of everything up to and including the input command
-        // before actually issuing the final '\n' that issues the command.
-        // This ensures that the buffer contents returned by this call contain
-        // only the command's output.
-        let out = self
-            .wait_for_serial_output(
-                self.guest_os.get_shell_prompt(),
-                Duration::from_secs(300),
-            )
-            .await?;
-
-        // Trim any leading newlines inserted when the command was issued and
-        // any trailing whitespace that isn't actually part of the command
-        // output. Any other embedded whitespace is the caller's problem.
-        Ok(out.trim().to_string())
+        Ok(())
     }
 
     /// Sends `string` to the guest's serial console worker, then waits for the

phd-tests/tests/src/boot_order.rs

@@ -389,8 +389,7 @@ async fn guest_can_adjust_boot_order(ctx: &Framework) {
     assert_eq!(new_boot_order, written_boot_order);
 
     // Now, reboot and check that the settings stuck.
-    vm.run_shell_command("reboot").await?;
-    vm.wait_to_boot().await?;
+    vm.graceful_reboot().await?;
 
     let boot_order_after_reboot = read_efivar(&vm, BOOT_ORDER_VAR).await?;
     assert_eq!(new_boot_order, boot_order_after_reboot);
@@ -469,8 +468,7 @@ async fn boot_order_source_priority(ctx: &Framework) {
         .await?
         .expect("unbootable was in the boot order");
 
-    vm_no_bootorder.run_shell_command("reboot").await?;
-    vm_no_bootorder.wait_to_boot().await?;
+    vm_no_bootorder.graceful_reboot().await?;
 
     let reloaded_order = read_efivar(&vm_no_bootorder, BOOT_ORDER_VAR).await?;
 
@@ -515,8 +513,7 @@ async fn boot_order_source_priority(ctx: &Framework) {
         .await?
         .expect("unbootable was in the boot order");
 
-    vm.run_shell_command("reboot").await?;
-    vm.wait_to_boot().await?;
+    vm.graceful_reboot().await?;
 
     let reloaded_order = read_efivar(&vm, BOOT_ORDER_VAR).await?;