From 300e72ab8a0d3b730dbafb3b5463f00615bbfc9b Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 25 Nov 2023 17:02:28 +0800 Subject: [PATCH] tests/robustness: init with powerfailure case Add `Robustness Test` pipeline for robustness test cases. Signed-off-by: Wei Fu --- .github/workflows/failpoint_test.yaml | 1 - .github/workflows/robustness_test.yaml | 18 +++ Makefile | 5 +- tests/dmflakey/dmflakey_test.go | 22 +-- tests/robustness/main_test.go | 17 +++ tests/robustness/powerfailure_test.go | 194 +++++++++++++++++++++++++ tests/utils/helpers.go | 26 ++++ 7 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/robustness_test.yaml create mode 100644 tests/robustness/main_test.go create mode 100644 tests/robustness/powerfailure_test.go create mode 100644 tests/utils/helpers.go diff --git a/.github/workflows/failpoint_test.yaml b/.github/workflows/failpoint_test.yaml index f5b79a9d4..46cafab6c 100644 --- a/.github/workflows/failpoint_test.yaml +++ b/.github/workflows/failpoint_test.yaml @@ -15,6 +15,5 @@ jobs: with: go-version: ${{ steps.goversion.outputs.goversion }} - run: | - sudo make root-test make gofail-enable make test-failpoint diff --git a/.github/workflows/robustness_test.yaml b/.github/workflows/robustness_test.yaml new file mode 100644 index 000000000..9aca5249e --- /dev/null +++ b/.github/workflows/robustness_test.yaml @@ -0,0 +1,18 @@ +name: Robustness Test +on: [push, pull_request] +permissions: read-all +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@v4 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: | + make gofail-enable + # build bbolt with failpoint + go install ./cmd/bbolt + sudo -E PATH=$PATH make test-robustness diff --git a/Makefile b/Makefile index bab533445..f43b25b20 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ test-failpoint: @echo "[failpoint] array freelist test" BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint -.PHONY: root-test # run tests that require root -root-test: +.PHONY: test-robustness # Running robustness tests requires root permission +test-robustness: go test -v ${TESTFLAGS} ./tests/dmflakey -test.root + go test -v ${TESTFLAGS} ./tests/robustness -test.root diff --git a/tests/dmflakey/dmflakey_test.go b/tests/dmflakey/dmflakey_test.go index 2cc1f8ea6..41c66db8d 100644 --- a/tests/dmflakey/dmflakey_test.go +++ b/tests/dmflakey/dmflakey_test.go @@ -12,35 +12,19 @@ import ( "testing" "time" + testutils "go.etcd.io/bbolt/tests/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) -var enableRoot bool - -func init() { - flag.BoolVar(&enableRoot, "test.root", false, "enable tests that require root") -} - func TestMain(m *testing.M) { flag.Parse() - requiresRoot() + testutils.RequiresRoot() os.Exit(m.Run()) } -func requiresRoot() { - if !enableRoot { - fmt.Fprintln(os.Stderr, "Skip tests that require root") - os.Exit(0) - } - - if os.Getuid() != 0 { - fmt.Fprintln(os.Stderr, "This test must be run as root.") - os.Exit(1) - } -} - func TestBasic(t *testing.T) { tmpDir := t.TempDir() diff --git a/tests/robustness/main_test.go b/tests/robustness/main_test.go new file mode 100644 index 000000000..d83f32700 --- /dev/null +++ b/tests/robustness/main_test.go @@ -0,0 +1,17 @@ +//go:build linux + +package robustness + +import ( + "flag" + "os" + "testing" + + testutils "go.etcd.io/bbolt/tests/utils" +) + +func TestMain(m *testing.M) { + flag.Parse() + testutils.RequiresRoot() + os.Exit(m.Run()) +} diff --git a/tests/robustness/powerfailure_test.go b/tests/robustness/powerfailure_test.go new file mode 100644 index 000000000..a1d0bc598 --- /dev/null +++ b/tests/robustness/powerfailure_test.go @@ -0,0 +1,194 @@ +//go:build linux + +package robustness + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "go.etcd.io/bbolt/tests/dmflakey" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// TestRestartFromPowerFailure is to test data after unexpected power failure. +func TestRestartFromPowerFailure(t *testing.T) { + flakey := initFlakeyDevice(t, t.Name(), dmflakey.FSTypeEXT4, "") + root := flakey.RootFS() + + dbPath := filepath.Join(root, "boltdb") + + args := []string{"bbolt", "bench", + "-work", // keep the database + "-path", dbPath, + "-count=1000000000", + "-batch-size=5", // separate total count into multiple truncation + } + + logPath := filepath.Join(t.TempDir(), fmt.Sprintf("%s.log", t.Name())) + logFd, err := os.Create(logPath) + require.NoError(t, err) + defer logFd.Close() + + fpURL := "127.0.0.1:12345" + + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdout = logFd + cmd.Stderr = logFd + cmd.Env = append(cmd.Env, "GOFAIL_HTTP="+fpURL) + t.Logf("start %s", strings.Join(args, " ")) + require.NoError(t, cmd.Start(), "args: %v", args) + + errCh := make(chan error, 1) + go func() { + errCh <- cmd.Wait() + }() + + defer func() { + if t.Failed() { + logData, err := os.ReadFile(logPath) + assert.NoError(t, err) + t.Logf("dump log:\n: %s", string(logData)) + } + }() + + time.Sleep(time.Duration(time.Now().UnixNano()%5+1) * time.Second) + t.Logf("simulate power failure") + + activeFailpoint(t, fpURL, "beforeSyncMetaPage", "panic") + + select { + case <-time.After(10 * time.Second): + t.Error("bbolt should stop with panic in seconds") + assert.NoError(t, cmd.Process.Kill()) + case err := <-errCh: + require.Error(t, err) + } + require.NoError(t, flakey.PowerFailure("")) + + st, err := os.Stat(dbPath) + require.NoError(t, err) + t.Logf("db size: %d", st.Size()) + + t.Logf("verify data") + output, err := exec.Command("bbolt", "check", dbPath).CombinedOutput() + require.NoError(t, err, "bbolt check output: %s", string(output)) +} + +// activeFailpoint actives the failpoint by http. +func activeFailpoint(t *testing.T, targetUrl string, fpName, fpVal string) { + u, err := url.Parse("http://" + path.Join(targetUrl, fpName)) + require.NoError(t, err, "parse url %s", targetUrl) + + req, err := http.NewRequest("PUT", u.String(), bytes.NewBuffer([]byte(fpVal))) + require.NoError(t, err) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode, "response body: %s", string(data)) +} + +// FlakeyDevice extends dmflakey.Flakey interface. +type FlakeyDevice interface { + // RootFS returns root filesystem. + RootFS() string + + // PowerFailure simulates power failure with drop all the writes. + PowerFailure(mntOpt string) error + + dmflakey.Flakey +} + +// initFlakeyDevice returns FlakeyDevice instance with a given filesystem. +func initFlakeyDevice(t *testing.T, name string, fsType dmflakey.FSType, mntOpt string) FlakeyDevice { + imgDir := t.TempDir() + + flakey, err := dmflakey.InitFlakey(name, imgDir, fsType) + require.NoError(t, err, "init flakey %s", name) + t.Cleanup(func() { + assert.NoError(t, flakey.Teardown()) + }) + + rootDir := t.TempDir() + err = unix.Mount(flakey.DevicePath(), rootDir, string(fsType), 0, mntOpt) + require.NoError(t, err, "init rootfs on %s", rootDir) + + t.Cleanup(func() { assert.NoError(t, unmountAll(rootDir)) }) + + return &flakeyT{ + Flakey: flakey, + + rootDir: rootDir, + mntOpt: mntOpt, + } +} + +type flakeyT struct { + dmflakey.Flakey + + rootDir string + mntOpt string +} + +// RootFS returns root filesystem. +func (f *flakeyT) RootFS() string { + return f.rootDir +} + +// PowerFailure simulates power failure with drop all the writes. +func (f *flakeyT) PowerFailure(mntOpt string) error { + if err := f.DropWrites(); err != nil { + return fmt.Errorf("failed to drop_writes: %w", err) + } + + if err := unmountAll(f.rootDir); err != nil { + return fmt.Errorf("failed to unmount rootfs %s: %w", f.rootDir, err) + } + + if mntOpt == "" { + mntOpt = f.mntOpt + } + + if err := f.AllowWrites(); err != nil { + return fmt.Errorf("failed to allow_writes: %w", err) + } + + if err := unix.Mount(f.DevicePath(), f.rootDir, string(f.Filesystem()), 0, mntOpt); err != nil { + return fmt.Errorf("failed to mount rootfs %s: %w", f.rootDir, err) + } + return nil +} + +func unmountAll(target string) error { + for i := 0; i < 50; i++ { + if err := unix.Unmount(target, 0); err != nil { + switch err { + case unix.EBUSY: + time.Sleep(500 * time.Millisecond) + continue + case unix.EINVAL: + return nil + default: + return fmt.Errorf("failed to umount %s: %w", target, err) + } + } + continue + } + return fmt.Errorf("failed to umount %s: %w", target, unix.EBUSY) +} diff --git a/tests/utils/helpers.go b/tests/utils/helpers.go new file mode 100644 index 000000000..f9c87f6e5 --- /dev/null +++ b/tests/utils/helpers.go @@ -0,0 +1,26 @@ +package utils + +import ( + "flag" + "fmt" + "os" +) + +var enableRoot bool + +func init() { + flag.BoolVar(&enableRoot, "test.root", false, "enable tests that require root") +} + +// RequiresRoot requires root and the test.root flag has been set. +func RequiresRoot() { + if !enableRoot { + fmt.Fprintln(os.Stderr, "Skip tests that require root") + os.Exit(0) + } + + if os.Getuid() != 0 { + fmt.Fprintln(os.Stderr, "This test must be run as root.") + os.Exit(1) + } +}