From 5e91ff156196757a95ebfd96c1b6e98fbb3fc82d Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 26 Jun 2024 17:44:38 -0700 Subject: [PATCH] Use ipfs/go-test for general test logic This replaces generic test code found in various locations in ipni. --- assigner/server/server_test.go | 180 ++++---------------- carstore/carwriter_test.go | 13 +- command/synthetic.go | 10 +- config/identity.go | 1 - e2e_test.go | 185 ++++++++++---------- go.mod | 1 + go.sum | 2 + internal/ingest/ingest_test.go | 6 +- internal/registry/apiconv_test.go | 8 +- internal/registry/registry_test.go | 8 +- server/find/extended_providers_test.go | 58 +++---- server/find/protocol_test.go | 14 +- server/find/server_test.go | 6 +- server/ingest/protocol_test.go | 10 +- test/testipnirunner.go | 224 ------------------------- test/typehelpers/typehelpers.go | 4 +- 16 files changed, 188 insertions(+), 542 deletions(-) delete mode 100644 test/testipnirunner.go diff --git a/assigner/server/server_test.go b/assigner/server/server_test.go index 110f8c181..dd7bb964d 100644 --- a/assigner/server/server_test.go +++ b/assigner/server/server_test.go @@ -1,22 +1,19 @@ package server_test import ( - "bufio" - "bytes" "context" "fmt" "net/http" "os" - "os/exec" "path/filepath" "runtime" - "strings" "testing" "time" "github.com/ipfs/go-cid" + testcmd "github.com/ipfs/go-test/cmd" + "github.com/ipfs/go-test/random" client "github.com/ipni/go-libipni/ingest/client" - "github.com/ipni/go-libipni/test" adminclient "github.com/ipni/storetheindex/admin/client" "github.com/ipni/storetheindex/assigner/config" "github.com/ipni/storetheindex/assigner/core" @@ -36,7 +33,10 @@ var pubIdent2 = sticfg.Identity{ PrivKey: "CAESQLtIPpQ0cFqLyK9Wnkd0J8lkslrd/g3KJSZOog7MLLt31PlBaXUpIIU5WaTuEJPioGK3+jEbDzFxDNrkQX5xKTg=", } -const pubsubTopic = "/indexer/ingest/mainnet" +const ( + indexerReadyMatch = "Indexer is ready" + pubsubTopic = "/indexer/ingest/mainnet" +) func setupServer(t *testing.T, assigner *core.Assigner) *server.Server { s, err := server.New("127.0.0.1:0", assigner) @@ -57,80 +57,38 @@ func TestAssignOnAnnounce(t *testing.T) { } t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - e := &e2eTestRunner{ - t: t, - dir: t.TempDir(), - ctx: ctx, - - indexerReady: make(chan struct{}, 1), - } - - // Use a clean environment, with the host's PATH, and a temporary HOME. - // We also tell "go install" to place binaries there. - hostEnv := os.Environ() - var filteredEnv []string - for _, env := range hostEnv { - if strings.Contains(env, "CC") || strings.Contains(env, "LDFLAGS") || strings.Contains(env, "CFLAGS") { - // Bring in the C compiler flags from the host. For example on a Nix - // machine, this compilation within the test will fail since the compiler - // will not find correct libraries. - filteredEnv = append(filteredEnv, env) - } else if strings.HasPrefix(env, "PATH") { - // Bring in the host's PATH. - filteredEnv = append(filteredEnv, env) - } - } - e.env = append(filteredEnv, []string{ - "HOME=" + e.dir, - "GOBIN=" + e.dir, - }...) - if runtime.GOOS == "windows" { - const gopath = "C:\\Projects\\Go" - err := os.MkdirAll(gopath, 0666) - require.NoError(t, err) - e.env = append(e.env, fmt.Sprintf("GOPATH=%s", gopath)) - } - t.Logf("Env: %s", strings.Join(e.env, " ")) - - // Reuse the host's build and module download cache. - // This should allow "go install" to reuse work. - for _, name := range []string{"GOCACHE", "GOMODCACHE"} { - out, err := exec.Command("go", "env", name).CombinedOutput() - require.NoError(t, err) - out = bytes.TrimSpace(out) - e.env = append(e.env, fmt.Sprintf("%s=%s", name, out)) - } + rnr := testcmd.NewRunner(t, t.TempDir()) cwd, err := os.Getwd() require.NoError(t, err) err = os.Chdir(filepath.Dir(filepath.Dir(cwd))) require.NoError(t, err) - e.run("go", "install", ".") + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + rnr.Run(ctx, "go", "install", ".") err = os.Chdir(cwd) require.NoError(t, err) - indexer := filepath.Join(e.dir, "storetheindex") - e.run(indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner", + indexer := filepath.Join(rnr.Dir, "storetheindex") + rnr.Run(ctx, indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner", "--listen-admin=/ip4/127.0.0.1/tcp/3602", "--listen-finder=/ip4/127.0.0.1/tcp/3600", "--listen-ingest=/ip4/127.0.0.1/tcp/3601", "--listen-p2p=/ip4/127.0.0.1/tcp/3603", ) - stiCfg, err := sticfg.Load(filepath.Join(e.dir, ".storetheindex", "config")) + stiCfg, err := sticfg.Load(filepath.Join(rnr.Dir, ".storetheindex", "config")) require.NoError(t, err) indexerID := stiCfg.Identity.PeerID t.Log("Initialized indexer", indexerID) - cmdIndexer := e.start(indexer, "daemon") - select { - case <-e.indexerReady: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer to start") - } + indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch) + cmdIndexer := rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady) + err = indexerReady.Wait(ctx) + require.NoError(t, err, "timed out waiting for indexer to start") // Assign a peer, to test that assigner reads this at startup. err = assign(ctx, "localhost:3602", pubIdent2.PeerID) @@ -156,7 +114,7 @@ func TestAssignOnAnnounce(t *testing.T) { require.NoError(t, err) ai.ID = peerID - mhs := test.RandomMultihashes(1) + mhs := random.Multihashes(1) assignChan, cancel := assigner.OnAssignment(peerID) defer cancel() @@ -173,29 +131,27 @@ func TestAssignOnAnnounce(t *testing.T) { } // Check assignment - outProvider := e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) + outProvider := rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) expect := peerID.String() require.Contains(t, string(outProvider), expect) - e.stop(cmdIndexer, 5*time.Second) + rnr.Stop(cmdIndexer, 5*time.Second) // Start index again to check that assignment was persisted. - cmdIndexer = e.start(indexer, "daemon") - select { - case <-e.indexerReady: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer to start") - } - outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) + cmdIndexer = rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady) + err = indexerReady.Wait(ctx) + require.NoError(t, err, "timed out waiting for indexer to start") + + outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) require.Contains(t, string(outProvider), expect) - outProvider = e.run(indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String()) + outProvider = rnr.Run(ctx, indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String()) require.Contains(t, string(outProvider), expect) - outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) + outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL) require.NotContains(t, string(outProvider), expect) - e.stop(cmdIndexer, 5*time.Second) + rnr.Stop(cmdIndexer, 5*time.Second) } // initAssigner initializes a new registry @@ -220,82 +176,6 @@ func initAssigner(t *testing.T, trustedID string) (*core.Assigner, config.Assign return assigner, cfg } -type e2eTestRunner struct { - t *testing.T - dir string - ctx context.Context - env []string - - indexerReady chan struct{} -} - -func (e *e2eTestRunner) run(name string, args ...string) []byte { - e.t.Helper() - - e.t.Logf("run: %s %s", name, strings.Join(args, " ")) - - cmd := exec.CommandContext(e.ctx, name, args...) - cmd.Env = e.env - out, err := cmd.CombinedOutput() - require.NoError(e.t, err, "err: %v, output: %s", err, out) - return out -} - -func (e *e2eTestRunner) start(prog string, args ...string) *exec.Cmd { - e.t.Helper() - - name := filepath.Base(prog) - e.t.Logf("run: %s %s", name, strings.Join(args, " ")) - - cmd := exec.CommandContext(e.ctx, prog, args...) - cmd.Env = e.env - - stdout, err := cmd.StdoutPipe() - require.NoError(e.t, err) - cmd.Stderr = cmd.Stdout - - scanner := bufio.NewScanner(stdout) - - go func() { - for scanner.Scan() { - line := scanner.Text() - - // Logging every single line via the test output is verbose, - // but helps see what's happening, especially when the test fails. - e.t.Logf("%s: %s", name, line) - if strings.Contains(line, "Indexer is ready") { - e.indexerReady <- struct{}{} - } - } - }() - - err = cmd.Start() - require.NoError(e.t, err) - return cmd -} - -func (e *e2eTestRunner) stop(cmd *exec.Cmd, timeout time.Duration) { - sig := os.Interrupt - if runtime.GOOS == "windows" { - // Windows can't send SIGINT. - sig = os.Kill - } - err := cmd.Process.Signal(sig) - require.NoError(e.t, err) - - waitErr := make(chan error, 1) - go func() { waitErr <- cmd.Wait() }() - - select { - case <-time.After(timeout): - e.t.Logf("killing command after %s: %s", timeout, cmd) - err := cmd.Process.Kill() - require.NoError(e.t, err) - case err := <-waitErr: - require.NoError(e.t, err) - } -} - func assign(ctx context.Context, indexer, peerIDStr string) error { cl, err := adminclient.New(indexer) if err != nil { diff --git a/carstore/carwriter_test.go b/carstore/carwriter_test.go index 0d5777f5b..4dc024877 100644 --- a/carstore/carwriter_test.go +++ b/carstore/carwriter_test.go @@ -9,18 +9,17 @@ import ( "testing" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-test/random" car "github.com/ipld/go-car/v2" carblockstore "github.com/ipld/go-car/v2/blockstore" carindex "github.com/ipld/go-car/v2/index" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/ingest/schema" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/carstore" "github.com/ipni/storetheindex/filestore" crypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - p2ptest "github.com/libp2p/go-libp2p/core/test" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" @@ -233,7 +232,7 @@ func newRandomLinkedList(t *testing.T, lsys ipld.LinkSystem, size int) (ipld.Lin var out []multihash.Multihash var nextLnk ipld.Link for i := 0; i < size; i++ { - mhs := test.RandomMultihashes(testEntriesChunkSize) + mhs := random.Multihashes(testEntriesChunkSize) chunk := &schema.EntryChunk{ Entries: mhs, Next: nextLnk, @@ -271,11 +270,7 @@ func mkProvLinkSystem(ds datastore.Datastore) ipld.LinkSystem { func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevLink ipld.Link, dstore datastore.Datastore) (ipld.Link, *schema.Advertisement, []multihash.Multihash, peer.ID, crypto.PrivKey) { lsys := mkProvLinkSystem(dstore) - priv, pubKey, err := p2ptest.RandTestKeyPair(crypto.Ed25519, 256) - require.NoError(t, err) - - p, err := peer.IDFromPublicKey(pubKey) - require.NoError(t, err) + p, priv, _ := random.Identity() ctxID := []byte("test-context-id") if metadata == nil { @@ -297,7 +292,7 @@ func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevL adv.Entries, mhs = newRandomLinkedList(t, lsys, eChunkCount) } - err = adv.Sign(priv) + err := adv.Sign(priv) require.NoError(t, err) node, err := adv.ToNode() diff --git a/command/synthetic.go b/command/synthetic.go index 00a009a78..9c2c4dbb3 100644 --- a/command/synthetic.go +++ b/command/synthetic.go @@ -9,7 +9,7 @@ import ( agg "github.com/filecoin-project/go-dagaggregator-unixfs" "github.com/ipfs/go-cid" - "github.com/ipni/go-libipni/test" + "github.com/ipfs/go-test/random" "github.com/urfave/cli/v2" ) @@ -131,7 +131,7 @@ func writeCidFile(fileName string, num int) error { for curr < num { if i == len(cids) { // Refil cids - cids = test.RandomCids(100) + cids = random.Cids(100) i = 0 } if _, err = w.WriteString(cids[i].String()); err != nil { @@ -171,7 +171,7 @@ func writeCidFileOfSize(fileName string, size int) error { for curr < size { if i == len(cids) { // Refil cids - cids = test.RandomCids(100) + cids = random.Cids(100) i = 0 } c := cids[i] @@ -212,7 +212,7 @@ func writeManifest(fileName string, num int) error { for curr < num { if i == len(cids) { // Refil cids - cids = test.RandomCids(100) + cids = random.Cids(100) i = 0 } @@ -257,7 +257,7 @@ func writeManifestOfSize(fileName string, size int) error { for curr < size { if i == len(cids) { // Refil cids - cids = test.RandomCids(100) + cids = random.Cids(100) i = 0 } c := cids[i] diff --git a/config/identity.go b/config/identity.go index eba5a52b0..8b8b2feda 100644 --- a/config/identity.go +++ b/config/identity.go @@ -30,7 +30,6 @@ type Identity struct { } func (i Identity) Decode() (peer.ID, ic.PrivKey, error) { - privKey, err := i.DecodePrivateKey("") if err != nil { return "", nil, fmt.Errorf("could not decode private key: %w", err) diff --git a/e2e_test.go b/e2e_test.go index bd7e015a3..71cec1691 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -16,16 +16,27 @@ import ( "testing" "time" + testcmd "github.com/ipfs/go-test/cmd" findclient "github.com/ipni/go-libipni/find/client" "github.com/ipni/go-libipni/find/model" "github.com/ipni/storetheindex/carstore" "github.com/ipni/storetheindex/config" "github.com/ipni/storetheindex/filestore" - "github.com/ipni/storetheindex/test" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" ) +const ( + // When this environ var is set to a value and running tests with -v flag, + // then TestIpniRunner output is logged. + envTestIpniRunnerOutput = "TEST_IPNI_RUNNER_OUTPUT" + + indexerReadyMatch = "Indexer is ready" + providerHasPeerMatch = "connected to peer successfully" + providerReadyMatch = "admin http server listening" + dhstoreReady = "Store opened." +) + // This is a full end-to-end test with storetheindex as the indexer daemon, // and index-provider/cmd/provider as a client. // We build both programs, noting that we always build the latest provider. @@ -66,41 +77,41 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() - e := test.NewTestIpniRunner(t, ctx, t.TempDir()) + rnr := testcmd.NewRunner(t, t.TempDir()) - carPath := filepath.Join(e.Dir, "sample-wrapped-v2.car") + carPath := filepath.Join(rnr.Dir, "sample-wrapped-v2.car") err := downloadFile("https://github.com/ipni/index-provider/raw/main/testdata/sample-wrapped-v2.car", carPath) require.NoError(t, err) // install storetheindex - indexer := filepath.Join(e.Dir, "storetheindex") - e.Run("go", "install", ".") + indexer := filepath.Join(rnr.Dir, "storetheindex") + rnr.Run(ctx, "go", "install", ".") - provider := filepath.Join(e.Dir, "provider") - dhstore := filepath.Join(e.Dir, "dhstore") - ipni := filepath.Join(e.Dir, "ipni") + provider := filepath.Join(rnr.Dir, "provider") + dhstore := filepath.Join(rnr.Dir, "dhstore") + ipni := filepath.Join(rnr.Dir, "ipni") cwd, err := os.Getwd() require.NoError(t, err) - err = os.Chdir(e.Dir) + err = os.Chdir(rnr.Dir) require.NoError(t, err) // install index-provider switch publisherProto { case "dtsync": // Install index-provider that supports dtsync. - e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@v0.13.6") + rnr.Run(ctx, "go", "install", "github.com/ipni/index-provider/cmd/provider@v0.13.6") case "libp2p", "libp2phttp", "http": - e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@latest") + rnr.Run(ctx, "go", "install", "github.com/ipni/index-provider/cmd/provider@latest") default: panic("providerProto must be one of: libp2phttp, http, dtsync") } // install dhstore - e.Run("go", "install", "-tags", "nofdb", "github.com/ipni/dhstore/cmd/dhstore@latest") + rnr.Run(ctx, "go", "install", "-tags", "nofdb", "github.com/ipni/dhstore/cmd/dhstore@latest") // install ipni-cli - e.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") + rnr.Run(ctx, "go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") err = os.Chdir(cwd) require.NoError(t, err) @@ -108,23 +119,23 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { // initialize index-provider switch publisherProto { case "dtsync": - e.Run(provider, "init") + rnr.Run(ctx, provider, "init") case "http": - e.Run(provider, "init", "--pubkind=http") + rnr.Run(ctx, provider, "init", "--pubkind=http") case "libp2p": - e.Run(provider, "init", "--pubkind=libp2p") + rnr.Run(ctx, provider, "init", "--pubkind=libp2p") case "libp2phttp": - e.Run(provider, "init", "--pubkind=libp2phttp") + rnr.Run(ctx, provider, "init", "--pubkind=libp2phttp") } - providerCfgPath := filepath.Join(e.Dir, ".index-provider", "config") + providerCfgPath := filepath.Join(rnr.Dir, ".index-provider", "config") cfg, err := config.Load(providerCfgPath) require.NoError(t, err) providerID := cfg.Identity.PeerID t.Logf("Initialized provider ID: %s", providerID) // initialize indexer - e.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") - stiCfgPath := filepath.Join(e.Dir, ".storetheindex", "config") + rnr.Run(ctx, indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") + stiCfgPath := filepath.Join(rnr.Dir, ".storetheindex", "config") cfg, err = config.Load(stiCfgPath) require.NoError(t, err) indexerID := cfg.Identity.PeerID @@ -134,63 +145,51 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { Storage: filestore.Config{ Type: "local", Local: filestore.LocalConfig{ - BasePath: e.Dir, + BasePath: rnr.Dir, }, }, } - rdMirrorDir := e.Dir + rdMirrorDir := rnr.Dir cfg.Save(stiCfgPath) // start provider - providerReady := test.NewStdoutWatcher(test.ProviderReadyMatch) - providerHasPeer := test.NewStdoutWatcher(test.ProviderHasPeerMatch) - cmdProvider := e.Start(test.NewExecution(provider, "daemon").WithWatcher(providerReady).WithWatcher(providerHasPeer)) - select { - case <-providerReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for provider to start") - } + providerReady := testcmd.NewStderrWatcher(providerReadyMatch) + providerHasPeer := testcmd.NewStderrWatcher(providerHasPeerMatch) + cmdProvider := rnr.Start(ctx, testcmd.Args(provider, "daemon"), providerReady, providerHasPeer) + err = providerReady.Wait(ctx) + require.NoError(t, err, "timed out waiting for provider to start") // start dhstore - dhstoreReady := test.NewStdoutWatcher(test.DhstoreReady) - cmdDhstore := e.Start(test.NewExecution(dhstore, "--storePath", e.Dir).WithWatcher(dhstoreReady)) - select { - case <-dhstoreReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for dhstore to start") - } + dhstoreReady := testcmd.NewStderrWatcher(dhstoreReady) + cmdDhstore := rnr.Start(ctx, testcmd.Args(dhstore, "--storePath", rnr.Dir), dhstoreReady) + err = dhstoreReady.Wait(ctx) + require.NoError(t, err, "timed out waiting for dhstore to start") // start indexer - indexerReady := test.NewStdoutWatcher(test.IndexerReadyMatch) - cmdIndexer := e.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady)) - select { - case <-indexerReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer to start") - } + indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch) + cmdIndexer := rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady) + err = indexerReady.Wait(ctx) + require.NoError(t, err, "timed out waiting for indexer to start") // connect provider to the indexer - e.Run(provider, "connect", + rnr.Run(ctx, provider, "connect", "--imaddr", fmt.Sprintf("/dns/localhost/tcp/3003/p2p/%s", indexerID), "--listen-admin", "http://localhost:3102", ) - select { - case <-providerHasPeer.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for provider to connect to indexer") - } + err = providerHasPeer.Wait(ctx) + require.NoError(t, err, "timed out waiting for provider to connect to indexer") // Allow provider advertisements, regardless of default policy. - e.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", providerID) + rnr.Run(ctx, indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", providerID) // Import a car file into the provider. This will cause the provider to // publish an advertisement that the indexer will read. The indexer will // then import the advertised content. - outImport := e.Run(provider, "import", "car", + outImport := rnr.Run(ctx, provider, "import", "car", "-i", carPath, "--listen-admin", "http://localhost:3102", ) - t.Logf("import output:\n%s\n", outImport) + t.Logf("import output:\n%s\n", string(outImport)) // Wait for the CAR to be indexed require.Eventually(t, func() bool { @@ -198,7 +197,7 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { "2DrjgbFdhNiSJghFWcQbzw6E8y4jU1Z7ZsWo3dJbYxwGTNFmAj", "2DrjgbFY1BnkgZwA3oL7ijiDn7sJMf4bhhQNTtDqgZP826vGzv", } { - findOutput := e.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) + findOutput := rnr.Run(ctx, ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) t.Logf("find output:\n%s\n", findOutput) if bytes.Contains(findOutput, []byte("not found")) { @@ -212,10 +211,10 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { return true }, 10*time.Second, time.Second) - e.Run("sync") + rnr.Run(ctx, "sync") // Check that ad was saved as CAR file. - dir, err := os.Open(e.Dir) + dir, err := os.Open(rnr.Dir) require.NoError(t, err) names, err := dir.Readdirnames(-1) dir.Close() @@ -233,19 +232,20 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { require.Equal(t, 1, carCount) require.Equal(t, 1, headCount) - outRates := e.Run(indexer, "admin", "telemetry", "-i", "http://localhost:3002") + outRates := rnr.Run(ctx, indexer, "admin", "telemetry", "-i", "http://localhost:3002") require.Contains(t, string(outRates), "1043 multihashes from 1 ads") t.Logf("Telemetry:\n%s", outRates) - root2 := filepath.Join(e.Dir, ".storetheindex2") - e.Env = append(e.Env, fmt.Sprintf("%s=%s", config.EnvDir, root2)) - e.Run(indexer, "init", "--store", "dhstore", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap", "--dhstore", "http://127.0.0.1:40080", + root2 := filepath.Join(rnr.Dir, ".storetheindex2") + rnr.Env = append(rnr.Env, fmt.Sprintf("%s=%s", config.EnvDir, root2)) + rnr.Run(ctx, indexer, "init", "--store", "dhstore", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap", "--dhstore", "http://127.0.0.1:40080", "--listen-admin", "/ip4/127.0.0.1/tcp/3202", "--listen-finder", "/ip4/127.0.0.1/tcp/3200", "--listen-ingest", "/ip4/127.0.0.1/tcp/3201", "--listen-p2p", "/ip4/127.0.0.1/tcp/3203") sti2CfgPath := filepath.Join(root2, "config") cfg, err = config.Load(sti2CfgPath) require.NoError(t, err) + indexer2ID := cfg.Identity.PeerID cfg.Ingest.AdvertisementMirror = config.Mirror{ Compress: "gzip", @@ -260,49 +260,43 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { Storage: filestore.Config{ Type: "local", Local: filestore.LocalConfig{ - BasePath: e.Dir, + BasePath: rnr.Dir, }, }, } cfg.Save(sti2CfgPath) - wrMirrorDir := e.Dir - - indexerReady2 := test.NewStdoutWatcher(test.IndexerReadyMatch) - cmdIndexer2 := e.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady2)) - select { - case <-indexerReady2.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer2 to start") - } + wrMirrorDir := rnr.Dir - outProviders := e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200") + indexerReady2 := testcmd.NewStdoutWatcher(indexerReadyMatch) + cmdIndexer2 := rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady2) + err = indexerReady2.Wait(ctx) + require.NoError(t, err, "timed out waiting for indexer2 to start") + + outProviders := rnr.Run(ctx, ipni, "provider", "--all", "--indexer", "http://localhost:3200") require.Contains(t, string(outProviders), "No providers registered with indexer", "expected no providers message") // import providers from first indexer. - e.Run(indexer, "admin", "import-providers", "--indexer", "http://localhost:3202", "--from", "localhost:3000") + rnr.Run(ctx, indexer, "admin", "import-providers", "--indexer", "http://localhost:3202", "--from", "localhost:3000") // Check that provider ID now appears in providers output. - outProviders = e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200", "--id-only") + outProviders = rnr.Run(ctx, ipni, "provider", "--all", "--indexer", "http://localhost:3200", "--id-only") require.Contains(t, string(outProviders), providerID, "expected provider id in providers output after import-providers") // Connect provider to the 2nd indexer. - e.Run(provider, "connect", + rnr.Run(ctx, provider, "connect", "--imaddr", fmt.Sprintf("/dns/localhost/tcp/3203/p2p/%s", indexer2ID), "--listen-admin", "http://localhost:3102", ) - select { - case <-providerHasPeer.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for provider to connect to indexer") - } + err = providerHasPeer.Wait(ctx) + require.NoError(t, err, "timed out waiting for provider to connect to indexer") // Tell provider to send direct announce to 2nd indexer. - out := e.Run(provider, "announce-http", + out := rnr.Run(ctx, provider, "announce-http", "-i", "http://localhost:3201", "--listen-admin", "http://localhost:3102", ) - t.Logf("announce output:\n%s\n", out) + t.Logf("announce output:\n%s\n", string(out)) // Create double hashed client and verify that 2nd indexer wrote // multihashes to dhstore. @@ -314,7 +308,7 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { var dhResp *model.FindResponse require.Eventually(t, func() bool { - dhResp, err = client.Find(e.Ctx, mh) + dhResp, err = client.Find(ctx, mh) return err == nil && len(dhResp.MultihashResults) != 0 }, 10*time.Second, time.Second) @@ -324,14 +318,14 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { require.Equal(t, providerID, dhResp.MultihashResults[0].ProviderResults[0].Provider.ID.String()) // Get the CAR file from the read mirror. - rdCarFile, err := carFromMirror(e.Ctx, rdMirrorDir) + rdCarFile, err := carFromMirror(ctx, rdMirrorDir) require.NoError(t, err) require.NotZero(t, rdCarFile.Size) // Get the CAR file from the write mirror and compare size. wrCarFS, err := filestore.NewLocal(wrMirrorDir) require.NoError(t, err) - wrCarFile, err := wrCarFS.Head(e.Ctx, rdCarFile.Path) + wrCarFile, err := wrCarFS.Head(ctx, rdCarFile.Path) require.NoError(t, err) require.Equal(t, rdCarFile.Size, wrCarFile.Size) t.Logf("CAR file %q is same size in read and write mirror: %d bytes", wrCarFile.Path, wrCarFile.Size) @@ -340,7 +334,7 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { // publish an advertisement that tells the indexer to remove the car file // content by contextID. The indexer will then import the advertisement and // remove content. - outRemove := e.Run(provider, "remove", "car", + outRemove := rnr.Run(ctx, provider, "remove", "car", "-i", carPath, "--listen-admin", "http://localhost:3102", ) @@ -352,7 +346,7 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { "2DrjgbFdhNiSJghFWcQbzw6E8y4jU1Z7ZsWo3dJbYxwGTNFmAj", "2DrjgbFY1BnkgZwA3oL7ijiDn7sJMf4bhhQNTtDqgZP826vGzv", } { - findOutput := e.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) + findOutput := rnr.Run(ctx, ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) t.Logf("find output:\n%s\n", findOutput) if !bytes.Contains(findOutput, []byte("not found")) { return false @@ -362,24 +356,24 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { }, 10*time.Second, time.Second) // Check that status is not frozen. - outStatus := e.Run(indexer, "admin", "status", "--indexer", "http://localhost:3202") + outStatus := rnr.Run(ctx, indexer, "admin", "status", "--indexer", "http://localhost:3202") require.Contains(t, string(outStatus), "Frozen: false", "expected indexer to be frozen") - e.Run(indexer, "admin", "freeze", "--indexer", "http://localhost:3202") - outProviders = e.Run(ipni, "provider", "--all", "--indexer", "http://localhost:3200") + rnr.Run(ctx, indexer, "admin", "freeze", "--indexer", "http://localhost:3202") + outProviders = rnr.Run(ctx, ipni, "provider", "--all", "--indexer", "http://localhost:3200") // Check that provider ID now appears as frozen in providers output. require.Contains(t, string(outProviders), "FrozenAtTime", "expected provider to be frozen") // Check that status is frozen. - outStatus = e.Run(indexer, "admin", "status", "--indexer", "http://localhost:3202") + outStatus = rnr.Run(ctx, indexer, "admin", "status", "--indexer", "http://localhost:3202") require.Contains(t, string(outStatus), "Frozen: true", "expected indexer to be frozen") logLevel := "info" if testing.Verbose() { logLevel = "debug" } - outgc := string(e.Run(indexer, "gc", "provider", "-pid", providerID, "-ll", logLevel, + outgc := string(rnr.Run(ctx, indexer, "gc", "provider", "-pid", providerID, "-ll", logLevel, "-i", "http://localhost:3200", "-i", "http://localhost:3000", "-sync-segment-size", "2", @@ -387,11 +381,10 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) { t.Logf("GC Results:\n%s\n", outgc) require.Contains(t, outgc, `"count": 1043, "total": 1043, "source": "CAR"`) - e.Stop(cmdIndexer2, time.Second) - - e.Stop(cmdIndexer, time.Second) - e.Stop(cmdProvider, time.Second) - e.Stop(cmdDhstore, time.Second) + rnr.Stop(cmdIndexer2, time.Second) + rnr.Stop(cmdIndexer, time.Second) + rnr.Stop(cmdProvider, time.Second) + rnr.Stop(cmdDhstore, time.Second) } func downloadFile(fileURL, filePath string) error { diff --git a/go.mod b/go.mod index 25e7317fb..9691e00ab 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 + github.com/ipfs/go-test v0.0.2 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 github.com/ipld/go-ipld-prime v0.21.0 diff --git a/go.sum b/go.sum index 7553ac96b..c5186a258 100644 --- a/go.sum +++ b/go.sum @@ -563,6 +563,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-test v0.0.2 h1:Wdxl4bKEdjEM8SLiulXMHlAQwHYOhX3CSBoUoEvncmM= +github.com/ipfs/go-test v0.0.2/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 655346dd8..6b29f25c6 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -16,6 +16,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-test/random" "github.com/ipld/go-ipld-prime" _ "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" @@ -31,7 +32,6 @@ import ( dstest "github.com/ipni/go-libipni/dagsync/test" schema "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/mautil" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/carstore" "github.com/ipni/storetheindex/config" "github.com/ipni/storetheindex/internal/registry" @@ -1790,7 +1790,7 @@ func newRandomLinkedList(t *testing.T, lsys ipld.LinkSystem, size int) (ipld.Lin var out []multihash.Multihash var nextLnk ipld.Link for i := 0; i < size; i++ { - mhs := test.RandomMultihashes(testEntriesChunkSize) + mhs := random.Multihashes(testEntriesChunkSize) chunk := &schema.EntryChunk{ Entries: mhs, Next: nextLnk, @@ -1898,7 +1898,7 @@ func publishAdvWithExtendedProviders(t *testing.T, epKeys[epID.String()] = epPriv adv.ExtendedProvider.Providers = append(adv.ExtendedProvider.Providers, schema.Provider{ ID: epID.String(), - Addresses: test.RandomAddrs(1), + Addresses: random.Addrs(1), Metadata: []byte(fmt.Sprintf("test-metadata-%d", i)), }) } diff --git a/internal/registry/apiconv_test.go b/internal/registry/apiconv_test.go index d1aaf3bde..6823abe81 100644 --- a/internal/registry/apiconv_test.go +++ b/internal/registry/apiconv_test.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ipfs/go-cid" - "github.com/ipni/go-libipni/test" + "github.com/ipfs/go-test/random" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" @@ -37,14 +37,14 @@ func TestRegToApiProviderInfo(t *testing.T) { frozenAtCid := cid.NewCidV1(cid.Raw, mh) frozenAtTime := lastAdTime.Add(-time.Hour) - maddrs := test.RandomMultiaddrs(2) + maddrs := random.Multiaddrs(2) ep1Addrs := maddrs[:1] ep2Addrs := maddrs[1:] epContextId := []byte("ep-context-id") - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Metadata := []byte("ep1-metadata") - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Metadata := []byte("ep2-metadata") regPI := ProviderInfo{ diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index 6f029f6ce..f3114ab30 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -12,8 +12,8 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" leveldb "github.com/ipfs/go-ds-leveldb" + "github.com/ipfs/go-test/random" "github.com/ipni/go-libipni/find/model" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/config" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -102,13 +102,13 @@ func TestDatastore(t *testing.T) { pubAddr, err := multiaddr.NewMultiaddr(publisherAddr) require.NoError(t, err) - maddrs := test.RandomMultiaddrs(2) + maddrs := random.Multiaddrs(2) ep1Addrs := maddrs[:1] ep2Addrs := maddrs[1:] epContextId := []byte("ep-context-id") - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Metadata := []byte("ep1-metadata") - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Metadata := []byte("ep2-metadata") provider2 := peer.AddrInfo{ ID: provID2, diff --git a/server/find/extended_providers_test.go b/server/find/extended_providers_test.go index 65c5c5928..40bee42a0 100644 --- a/server/find/extended_providers_test.go +++ b/server/find/extended_providers_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-test/random" "github.com/ipni/go-indexer-core" "github.com/ipni/go-libipni/find/client" "github.com/ipni/go-libipni/find/model" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/internal/registry" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -17,13 +17,13 @@ import ( ) func providersShouldBeUnaffectedByExtendedProvidersOfEachOtherTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - provider1Id, _, _ := test.RandomIdentity() + provider1Id, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") metadata1 := []byte("test-metadata-1") - maddrs := test.RandomMultiaddrs(3) + maddrs := random.Multiaddrs(3) addrs1 := maddrs[:1] ep1Addrs := maddrs[1:2] - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() createProviderAndPopulateIndexer(t, ctx, ind, reg, ctxId1, metadata1, provider1Id, addrs1, ®istry.ExtendedProviders{ Providers: []registry.ExtendedProviderInfo{ { @@ -33,7 +33,7 @@ func providersShouldBeUnaffectedByExtendedProvidersOfEachOtherTest(ctx context.C }, }) - provider2Id, _, _ := test.RandomIdentity() + provider2Id, _, _ := random.Identity() ctxId2 := []byte("test-context-id-2") metadata2 := []byte("test-metadata-2") addrs2 := maddrs[2:3] @@ -55,17 +55,17 @@ func providersShouldBeUnaffectedByExtendedProvidersOfEachOtherTest(ctx context.C } func extendedProviderShouldHaveOwnMetadataTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - provider1Id, _, _ := test.RandomIdentity() + provider1Id, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") metadata1 := []byte("test-metadata-1") - maddrs := test.RandomMultiaddrs(3) + maddrs := random.Multiaddrs(3) addrs1 := maddrs[:1] - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Addrs := maddrs[1:2] ep1Metadata := []byte("test-metadata-ep1") - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Addrs := maddrs[2:3] ep2Metadata := []byte("test-metadata-ep2") prov1, mhs1 := createProviderAndPopulateIndexer(t, ctx, ind, reg, ctxId1, metadata1, provider1Id, addrs1, ®istry.ExtendedProviders{ @@ -122,15 +122,15 @@ func extendedProviderShouldHaveOwnMetadataTest(ctx context.Context, t *testing.T } func extendedProviderShouldInheritMetadataOfMainProviderTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - provider1Id, _, _ := test.RandomIdentity() + provider1Id, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") metadata1 := []byte("test-metadata-1") - maddrs := test.RandomMultiaddrs(3) + maddrs := random.Multiaddrs(3) addrs1 := maddrs[:1] - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Addrs := maddrs[1:2] - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Addrs := maddrs[2:3] prov1, mhs1 := createProviderAndPopulateIndexer(t, ctx, ind, reg, ctxId1, metadata1, provider1Id, addrs1, ®istry.ExtendedProviders{ @@ -185,16 +185,16 @@ func extendedProviderShouldInheritMetadataOfMainProviderTest(ctx context.Context } func contextualExtendedProvidersShouldUnionUpWithChainLevelOnesTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - provider1Id, _, _ := test.RandomIdentity() + provider1Id, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") metadata1 := []byte("test-metadata-1") - maddrs := test.RandomMultiaddrs(3) + maddrs := random.Multiaddrs(3) addrs1 := maddrs[:1] - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Addrs := maddrs[1:2] - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Addrs := maddrs[2:3] prov1, mhs1 := createProviderAndPopulateIndexer(t, ctx, ind, reg, ctxId1, metadata1, provider1Id, addrs1, ®istry.ExtendedProviders{ @@ -217,7 +217,7 @@ func contextualExtendedProvidersShouldUnionUpWithChainLevelOnesTest(ctx context. }) ctxId2 := []byte("test-context-id-2") - mhs2 := test.RandomMultihashes(10) + mhs2 := random.Multihashes(10) v := indexer.Value{ ProviderID: prov1, @@ -281,16 +281,16 @@ func contextualExtendedProvidersShouldUnionUpWithChainLevelOnesTest(ctx context. } func contextualExtendedProvidersShouldOverrideChainLevelOnesTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - provider1Id, _, _ := test.RandomIdentity() + provider1Id, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") metadata1 := []byte("test-metadata-1") - maddrs := test.RandomMultiaddrs(3) + maddrs := random.Multiaddrs(3) addrs1 := maddrs[:1] - ep1, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() ep1Addrs := maddrs[1:2] - ep2, _, _ := test.RandomIdentity() + ep2, _, _ := random.Identity() ep2Addrs := maddrs[2:3] prov1, mhs1 := createProviderAndPopulateIndexer(t, ctx, ind, reg, ctxId1, metadata1, provider1Id, addrs1, ®istry.ExtendedProviders{ @@ -336,11 +336,11 @@ func contextualExtendedProvidersShouldOverrideChainLevelOnesTest(ctx context.Con } func mainProviderChainRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - providerId, _, _ := test.RandomIdentity() + providerId, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") providerMetadata := []byte("provider metadata") chainMetadata := []byte("chain level metadata") - maddrs := test.RandomMultiaddrs(4) + maddrs := random.Multiaddrs(4) addrs := maddrs[0:2] chainAddrs := maddrs[2:4] @@ -358,7 +358,7 @@ func mainProviderChainRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context.C { PeerID: providerId, Metadata: providerMetadata, - Addrs: test.RandomMultiaddrs(2), + Addrs: random.Multiaddrs(2), }, }, }}, @@ -388,11 +388,11 @@ func mainProviderChainRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context.C } func mainProviderContextRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { - providerId, _, _ := test.RandomIdentity() + providerId, _, _ := random.Identity() ctxId1 := []byte("test-context-id-1") providerMetadata := []byte("provider metadata") contextMetadata := []byte("context level metadata") - maddrs := test.RandomMultiaddrs(4) + maddrs := random.Multiaddrs(4) addrs := maddrs[0:2] contextAddrs := maddrs[2:4] @@ -401,7 +401,7 @@ func mainProviderContextRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context { PeerID: providerId, Metadata: providerMetadata, - Addrs: test.RandomMultiaddrs(2), + Addrs: random.Multiaddrs(2), }, }, ContextualProviders: map[string]registry.ContextualExtendedProviders{string(ctxId1): { @@ -441,7 +441,7 @@ func mainProviderContextRecordIsIncludedIfItsMetadataIsDifferentTest(ctx context func createProviderAndPopulateIndexer(t *testing.T, ctx context.Context, ind indexer.Interface, reg *registry.Registry, contextID []byte, metadata []byte, providerID peer.ID, addrs []multiaddr.Multiaddr, extendedProviders *registry.ExtendedProviders) (peer.ID, []multihash.Multihash) { // Generate some multihashes and populate indexer - mhs := test.RandomMultihashes(10) + mhs := random.Multihashes(10) v := indexer.Value{ ProviderID: providerID, diff --git a/server/find/protocol_test.go b/server/find/protocol_test.go index 148c9683d..e4c17f4e0 100644 --- a/server/find/protocol_test.go +++ b/server/find/protocol_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/ipfs/go-test/random" indexer "github.com/ipni/go-indexer-core" "github.com/ipni/go-indexer-core/cache/radixcache" "github.com/ipni/go-indexer-core/engine" @@ -16,7 +17,6 @@ import ( "github.com/ipni/go-indexer-core/store/pebble" "github.com/ipni/go-libipni/find/client" "github.com/ipni/go-libipni/find/model" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/config" "github.com/ipni/storetheindex/internal/registry" httpserver "github.com/ipni/storetheindex/server/find" @@ -228,7 +228,7 @@ func populateIndex(ind indexer.Interface, mhs []multihash.Multihash, v indexer.V func findIndexTest(ctx context.Context, t *testing.T, f client.Finder, ind indexer.Interface, reg *registry.Registry) { // Generate some multihashes and populate indexer - mhs := test.RandomMultihashes(2) + mhs := random.Multihashes(2) p, err := peer.Decode(providerID) require.NoError(t, err) ctxID := []byte("test-context-id") @@ -355,7 +355,7 @@ func verifyProviderInfo(t *testing.T, provInfo *model.ProviderInfo) { func removeProviderTest(ctx context.Context, t *testing.T, c *client.Client, ind indexer.Interface, reg *registry.Registry) { // Generate some multihashes and populate indexer - mhs := test.RandomMultihashes(15) + mhs := random.Multihashes(15) p, err := peer.Decode(providerID) require.NoError(t, err) ctxID := []byte("test-context-id") @@ -410,7 +410,7 @@ func removeProviderTest(ctx context.Context, t *testing.T, c *client.Client, ind func getStatsTest(ctx context.Context, t *testing.T, ind indexer.Interface, refreshStats func(), c *client.Client) { t.Parallel() - mhs := test.RandomMultihashes(15) + mhs := random.Multihashes(15) p, err := peer.Decode(providerID) require.NoError(t, err) ctxID := []byte("test-context-id") @@ -438,15 +438,15 @@ func register(ctx context.Context, t *testing.T, reg *registry.Registry) peer.ID maddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/9999") require.NoError(t, err) - ep1, _, _ := test.RandomIdentity() - ep2, _, _ := test.RandomIdentity() + ep1, _, _ := random.Identity() + ep2, _, _ := random.Identity() provider := peer.AddrInfo{ ID: peerID, Addrs: []multiaddr.Multiaddr{maddr}, } - maddrs := test.RandomMultiaddrs(2) + maddrs := random.Multiaddrs(2) provAddrs := maddrs[:1] ctxAddrs := maddrs[1:] diff --git a/server/find/server_test.go b/server/find/server_test.go index 9bed785f9..98da4f42c 100644 --- a/server/find/server_test.go +++ b/server/find/server_test.go @@ -11,8 +11,8 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-test/random" "github.com/ipni/go-indexer-core" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/server/find" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -57,7 +57,7 @@ iframe { ` func TestServer_CORSWithExpectedContentType(t *testing.T) { - mhs := test.RandomMultihashes(10) + mhs := random.Multihashes(10) p, err := peer.Decode("12D3KooWKRyzVWW6ChFjQjK4miCty85Niy48tpPV95XdKu1BcvMA") require.NoError(t, err) c := cid.NewCidV1(cid.Raw, mhs[0]) @@ -164,7 +164,7 @@ func TestServer_StreamingResponse(t *testing.T) { landing = strings.ReplaceAll(landingRendered, "\n", "\r\n") } - mhs := test.RandomMultihashes(10) + mhs := random.Multihashes(10) p, err := peer.Decode("12D3KooWKRyzVWW6ChFjQjK4miCty85Niy48tpPV95XdKu1BcvMA") require.NoError(t, err) diff --git a/server/ingest/protocol_test.go b/server/ingest/protocol_test.go index 3dddc913d..c9ad8daa5 100644 --- a/server/ingest/protocol_test.go +++ b/server/ingest/protocol_test.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-test/random" indexer "github.com/ipni/go-indexer-core" "github.com/ipni/go-indexer-core/engine" "github.com/ipni/go-indexer-core/store/memory" @@ -17,7 +18,6 @@ import ( "github.com/ipni/go-libipni/announce/httpsender" "github.com/ipni/go-libipni/announce/message" "github.com/ipni/go-libipni/ingest/client" - "github.com/ipni/go-libipni/test" "github.com/ipni/storetheindex/config" "github.com/ipni/storetheindex/internal/ingest" "github.com/ipni/storetheindex/internal/registry" @@ -154,7 +154,7 @@ func announceTest(t *testing.T, peerID peer.ID, sender announce.Sender) { p2pAddrs, err := peer.AddrInfoToP2pAddrs(ai) require.NoError(t, err) - mhs := test.RandomMultihashes(1) + mhs := random.Multihashes(1) msg := message.Message{ Cid: cid.NewCidV1(22, mhs[0]), @@ -190,7 +190,7 @@ func indexContent(t *testing.T, cl client.Interface, providerID peer.ID, private ctx, cancel := context.WithCancel(context.Background()) defer cancel() - mhs := test.RandomMultihashes(1) + mhs := random.Multihashes(1) contextID := []byte("test-context-id") metadata := []byte("test-metadata") @@ -222,7 +222,7 @@ func indexContentNewAddr(t *testing.T, cl client.Interface, providerID peer.ID, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - mhs := test.RandomMultihashes(1) + mhs := random.Multihashes(1) ctxID := []byte("test-context-id") metadata := []byte("test-metadata") @@ -245,7 +245,7 @@ func indexContentFail(t *testing.T, cl client.Interface, providerID peer.ID, pri ctx, cancel := context.WithCancel(context.Background()) defer cancel() - mhs := test.RandomMultihashes(1) + mhs := random.Multihashes(1) contextID := make([]byte, schema.MaxContextIDLen+1) metadata := []byte("test-metadata") diff --git a/test/testipnirunner.go b/test/testipnirunner.go deleted file mode 100644 index 8f7e58ba2..000000000 --- a/test/testipnirunner.go +++ /dev/null @@ -1,224 +0,0 @@ -package test - -import ( - "bufio" - "bytes" - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -const ( - // When this environ var is set to a value and running tests with -v flag, - // then TestIpniRunner output is logged. - EnvTestIpniRunnerOutput = "TEST_IPNI_RUNNER_OUTPUT" - - IndexerReadyMatch = "Indexer is ready" - ProviderHasPeerMatch = "connected to peer successfully" - ProviderReadyMatch = "admin http server listening" - DhstoreReady = "Store opened." -) - -// StdoutWatcher is a helper for watching the stdout of a command for a -// specific string. It is used by TestIpniRunner to watch for specific -// output from the commands. The Signal channel will be sent on when the -// match string is found. -type StdoutWatcher struct { - Match string - Signal chan struct{} -} - -func NewStdoutWatcher(match string) StdoutWatcher { - return StdoutWatcher{ - Match: match, - Signal: make(chan struct{}, 1), - } -} - -// TestIpniRunner is a helper for running the indexer and other commands. -// TestIpniRunner is not specifically tied to the indexer, but is designed -// to be used to manage multiple processes in a test; and is therefore useful -// for testing the indexer, the dhstore, and providers, all in a temporary -// directory and with a test environment. -type TestIpniRunner struct { - t *testing.T - verbose bool - - Ctx context.Context - Dir string - Env []string -} - -// NewTestIpniRunner creates a new TestIpniRunner for the given test, -// context, and temporary directory. It also takes a list of StdoutWatchers, -// which will be used to watch for specific output from the commands. -func NewTestIpniRunner(t *testing.T, ctx context.Context, dir string) *TestIpniRunner { - tr := TestIpniRunner{ - t: t, - verbose: os.Getenv(EnvTestIpniRunnerOutput) != "", - - Ctx: ctx, - Dir: dir, - } - - // Use a clean environment, with the host's PATH, and a temporary HOME. - // We also tell "go install" to place binaries there. - hostEnv := os.Environ() - var filteredEnv []string - for _, env := range hostEnv { - if strings.Contains(env, "CC") || strings.Contains(env, "LDFLAGS") || strings.Contains(env, "CFLAGS") { - // Bring in the C compiler flags from the host. For example on a Nix - // machine, this compilation within the test will fail since the compiler - // will not find correct libraries. - filteredEnv = append(filteredEnv, env) - } else if strings.HasPrefix(env, "PATH") { - // Bring in the host's PATH. - filteredEnv = append(filteredEnv, env) - } - } - tr.Env = append(filteredEnv, []string{ - "HOME=" + tr.Dir, - "GOBIN=" + tr.Dir, - }...) - if runtime.GOOS == "windows" { - const gopath = "C:\\Projects\\Go" - err := os.MkdirAll(gopath, 0666) - require.NoError(t, err) - tr.Env = append(tr.Env, fmt.Sprintf("GOPATH=%s", gopath)) - } - if tr.verbose { - t.Logf("Env: %s", strings.Join(tr.Env, " ")) - } - - // Reuse the host's build and module download cache. - // This should allow "go install" to reuse work. - for _, name := range []string{"GOCACHE", "GOMODCACHE"} { - out, err := exec.Command("go", "env", name).CombinedOutput() - require.NoError(t, err) - out = bytes.TrimSpace(out) - tr.Env = append(tr.Env, fmt.Sprintf("%s=%s", name, out)) - } - - return &tr -} - -// Run runs a command and returns its output. This is useful for executing -// synchronous commands within the temporary environment. -func (tr *TestIpniRunner) Run(name string, args ...string) []byte { - tr.t.Helper() - - if tr.verbose { - tr.t.Logf("run: %s %s", name, strings.Join(args, " ")) - } - - cmd := exec.CommandContext(tr.Ctx, name, args...) - cmd.Env = tr.Env - out, err := cmd.CombinedOutput() - require.NoError(tr.t, err, string(out)) - return out -} - -type Execution struct { - Name string - Args []string - Watchers []StdoutWatcher -} - -func NewExecution(name string, args ...string) Execution { - return Execution{ - Name: name, - Args: args, - Watchers: []StdoutWatcher{}, - } -} - -func (p Execution) String() string { - return p.Name + " " + strings.Join(p.Args, " ") -} - -func (p Execution) WithWatcher(watcher StdoutWatcher) Execution { - p.Watchers = append(append([]StdoutWatcher{}, p.Watchers...), watcher) - return p -} - -// Start starts a command and returns the command. This is useful for executing -// asynchronous commands within the temporary environment. It will watch the -// command's stdout for the given match string, and send on a watcher's -// channel when/if found. -func (tr *TestIpniRunner) Start(ex Execution) *exec.Cmd { - tr.t.Helper() - - name := filepath.Base(ex.Name) - if tr.verbose { - tr.t.Logf("run: %s", ex.String()) - } - - cmd := exec.CommandContext(tr.Ctx, ex.Name, ex.Args...) - cmd.Env = tr.Env - - stdout, err := cmd.StdoutPipe() - require.NoError(tr.t, err) - cmd.Stderr = cmd.Stdout - - scanner := bufio.NewScanner(stdout) - - if tr.verbose { - for _, watcher := range ex.Watchers { - tr.t.Logf("watching: %s for [%s]", name, watcher.Match) - } - } - go func() { - for scanner.Scan() { - line := strings.ToLower(scanner.Text()) - - if tr.verbose { - // Logging every single line via the test output is verbose, - // but helps see what's happening, especially when the test fails. - tr.t.Logf("%s: %s", name, line) - } - - for _, watcher := range ex.Watchers { - if strings.Contains(line, strings.ToLower(watcher.Match)) { - watcher.Signal <- struct{}{} - } - } - } - }() - - err = cmd.Start() - require.NoError(tr.t, err) - return cmd -} - -// Stop stops a command. It sends SIGINT, and if that doesn't work, SIGKILL. -func (tr *TestIpniRunner) Stop(cmd *exec.Cmd, timeout time.Duration) { - sig := os.Interrupt - if runtime.GOOS == "windows" { - // Windows can't send SIGINT. - sig = os.Kill - } - err := cmd.Process.Signal(sig) - require.NoError(tr.t, err) - - waitErr := make(chan error, 1) - go func() { waitErr <- cmd.Wait() }() - - select { - case <-time.After(timeout): - tr.t.Logf("killing command after %s: %s", timeout, cmd) - err = cmd.Process.Kill() - require.NoError(tr.t, err) - case err = <-waitErr: - if runtime.GOOS != "windows" { - require.NoError(tr.t, err) - } - } -} diff --git a/test/typehelpers/typehelpers.go b/test/typehelpers/typehelpers.go index e1ce1db04..db67a5a3f 100644 --- a/test/typehelpers/typehelpers.go +++ b/test/typehelpers/typehelpers.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "testing" + "github.com/ipfs/go-test/random" hamt "github.com/ipld/go-ipld-adl-hamt" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" @@ -16,7 +17,6 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/ipni/go-libipni/ingest/schema" - "github.com/ipni/go-libipni/test" crypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multihash" @@ -145,7 +145,7 @@ func (b RandomEntryChunkBuilder) Build(t *testing.T, lsys ipld.LinkSystem) datam mhs = append(mhs, badmh) } } else { - mhs = test.RandomMultihashes(int(b.EntriesPerChunk)) + mhs = random.Multihashes(int(b.EntriesPerChunk)) } var err error