Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use ipfs/go-test for general test logic #2633

Merged
merged 1 commit into from
Jun 28, 2024
Merged

Use ipfs/go-test for general test logic #2633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 30 additions & 150 deletions assigner/server/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
package server_test

import (
"bufio"
"bytes"
"context"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"

"github.com/ipfs/go-cid"
testcmd "github.com/ipfs/go-test/cmd"
"github.com/ipfs/go-test/random"
client "github.com/ipni/go-libipni/ingest/client"
"github.com/ipni/go-libipni/test"
adminclient "github.com/ipni/storetheindex/admin/client"
"github.com/ipni/storetheindex/assigner/config"
"github.com/ipni/storetheindex/assigner/core"
Expand All @@ -36,7 +33,10 @@ var pubIdent2 = sticfg.Identity{
PrivKey: "CAESQLtIPpQ0cFqLyK9Wnkd0J8lkslrd/g3KJSZOog7MLLt31PlBaXUpIIU5WaTuEJPioGK3+jEbDzFxDNrkQX5xKTg=",
}

const pubsubTopic = "/indexer/ingest/mainnet"
const (
indexerReadyMatch = "Indexer is ready"
pubsubTopic = "/indexer/ingest/mainnet"
)

func setupServer(t *testing.T, assigner *core.Assigner) *server.Server {
s, err := server.New("127.0.0.1:0", assigner)
Expand All @@ -57,80 +57,38 @@ func TestAssignOnAnnounce(t *testing.T) {
}
t.Parallel()

ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
e := &e2eTestRunner{
t: t,
dir: t.TempDir(),
ctx: ctx,

indexerReady: make(chan struct{}, 1),
}

// Use a clean environment, with the host's PATH, and a temporary HOME.
// We also tell "go install" to place binaries there.
hostEnv := os.Environ()
var filteredEnv []string
for _, env := range hostEnv {
if strings.Contains(env, "CC") || strings.Contains(env, "LDFLAGS") || strings.Contains(env, "CFLAGS") {
// Bring in the C compiler flags from the host. For example on a Nix
// machine, this compilation within the test will fail since the compiler
// will not find correct libraries.
filteredEnv = append(filteredEnv, env)
} else if strings.HasPrefix(env, "PATH") {
// Bring in the host's PATH.
filteredEnv = append(filteredEnv, env)
}
}
e.env = append(filteredEnv, []string{
"HOME=" + e.dir,
"GOBIN=" + e.dir,
}...)
if runtime.GOOS == "windows" {
const gopath = "C:\\Projects\\Go"
err := os.MkdirAll(gopath, 0666)
require.NoError(t, err)
e.env = append(e.env, fmt.Sprintf("GOPATH=%s", gopath))
}
t.Logf("Env: %s", strings.Join(e.env, " "))

// Reuse the host's build and module download cache.
// This should allow "go install" to reuse work.
for _, name := range []string{"GOCACHE", "GOMODCACHE"} {
out, err := exec.Command("go", "env", name).CombinedOutput()
require.NoError(t, err)
out = bytes.TrimSpace(out)
e.env = append(e.env, fmt.Sprintf("%s=%s", name, out))
}
rnr := testcmd.NewRunner(t, t.TempDir())

cwd, err := os.Getwd()
require.NoError(t, err)

err = os.Chdir(filepath.Dir(filepath.Dir(cwd)))
require.NoError(t, err)
e.run("go", "install", ".")

ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()

rnr.Run(ctx, "go", "install", ".")

err = os.Chdir(cwd)
require.NoError(t, err)

indexer := filepath.Join(e.dir, "storetheindex")
e.run(indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner",
indexer := filepath.Join(rnr.Dir, "storetheindex")
rnr.Run(ctx, indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner",
"--listen-admin=/ip4/127.0.0.1/tcp/3602",
"--listen-finder=/ip4/127.0.0.1/tcp/3600",
"--listen-ingest=/ip4/127.0.0.1/tcp/3601",
"--listen-p2p=/ip4/127.0.0.1/tcp/3603",
)
stiCfg, err := sticfg.Load(filepath.Join(e.dir, ".storetheindex", "config"))
stiCfg, err := sticfg.Load(filepath.Join(rnr.Dir, ".storetheindex", "config"))
require.NoError(t, err)
indexerID := stiCfg.Identity.PeerID
t.Log("Initialized indexer", indexerID)

cmdIndexer := e.start(indexer, "daemon")
select {
case <-e.indexerReady:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}
indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch)
cmdIndexer := rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady)
err = indexerReady.Wait(ctx)
require.NoError(t, err, "timed out waiting for indexer to start")

// Assign a peer, to test that assigner reads this at startup.
err = assign(ctx, "localhost:3602", pubIdent2.PeerID)
Expand All @@ -156,7 +114,7 @@ func TestAssignOnAnnounce(t *testing.T) {
require.NoError(t, err)
ai.ID = peerID

mhs := test.RandomMultihashes(1)
mhs := random.Multihashes(1)

assignChan, cancel := assigner.OnAssignment(peerID)
defer cancel()
Expand All @@ -173,29 +131,27 @@ func TestAssignOnAnnounce(t *testing.T) {
}

// Check assignment
outProvider := e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
outProvider := rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
expect := peerID.String()
require.Contains(t, string(outProvider), expect)

e.stop(cmdIndexer, 5*time.Second)
rnr.Stop(cmdIndexer, 5*time.Second)

// Start index again to check that assignment was persisted.
cmdIndexer = e.start(indexer, "daemon")
select {
case <-e.indexerReady:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}
outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
cmdIndexer = rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady)
err = indexerReady.Wait(ctx)
require.NoError(t, err, "timed out waiting for indexer to start")

outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
require.Contains(t, string(outProvider), expect)

outProvider = e.run(indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String())
outProvider = rnr.Run(ctx, indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String())
require.Contains(t, string(outProvider), expect)

outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
require.NotContains(t, string(outProvider), expect)

e.stop(cmdIndexer, 5*time.Second)
rnr.Stop(cmdIndexer, 5*time.Second)
}

// initAssigner initializes a new registry
Expand All @@ -220,82 +176,6 @@ func initAssigner(t *testing.T, trustedID string) (*core.Assigner, config.Assign
return assigner, cfg
}

type e2eTestRunner struct {
t *testing.T
dir string
ctx context.Context
env []string

indexerReady chan struct{}
}

func (e *e2eTestRunner) run(name string, args ...string) []byte {
e.t.Helper()

e.t.Logf("run: %s %s", name, strings.Join(args, " "))

cmd := exec.CommandContext(e.ctx, name, args...)
cmd.Env = e.env
out, err := cmd.CombinedOutput()
require.NoError(e.t, err, "err: %v, output: %s", err, out)
return out
}

func (e *e2eTestRunner) start(prog string, args ...string) *exec.Cmd {
e.t.Helper()

name := filepath.Base(prog)
e.t.Logf("run: %s %s", name, strings.Join(args, " "))

cmd := exec.CommandContext(e.ctx, prog, args...)
cmd.Env = e.env

stdout, err := cmd.StdoutPipe()
require.NoError(e.t, err)
cmd.Stderr = cmd.Stdout

scanner := bufio.NewScanner(stdout)

go func() {
for scanner.Scan() {
line := scanner.Text()

// Logging every single line via the test output is verbose,
// but helps see what's happening, especially when the test fails.
e.t.Logf("%s: %s", name, line)
if strings.Contains(line, "Indexer is ready") {
e.indexerReady <- struct{}{}
}
}
}()

err = cmd.Start()
require.NoError(e.t, err)
return cmd
}

func (e *e2eTestRunner) stop(cmd *exec.Cmd, timeout time.Duration) {
sig := os.Interrupt
if runtime.GOOS == "windows" {
// Windows can't send SIGINT.
sig = os.Kill
}
err := cmd.Process.Signal(sig)
require.NoError(e.t, err)

waitErr := make(chan error, 1)
go func() { waitErr <- cmd.Wait() }()

select {
case <-time.After(timeout):
e.t.Logf("killing command after %s: %s", timeout, cmd)
err := cmd.Process.Kill()
require.NoError(e.t, err)
case err := <-waitErr:
require.NoError(e.t, err)
}
}

func assign(ctx context.Context, indexer, peerIDStr string) error {
cl, err := adminclient.New(indexer)
if err != nil {
Expand Down
13 changes: 4 additions & 9 deletions carstore/carwriter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,17 @@ import (
"testing"

"github.com/ipfs/go-datastore"
"github.com/ipfs/go-test/random"
car "github.com/ipld/go-car/v2"
carblockstore "github.com/ipld/go-car/v2/blockstore"
carindex "github.com/ipld/go-car/v2/index"
"github.com/ipld/go-ipld-prime"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/ipni/go-libipni/ingest/schema"
"github.com/ipni/go-libipni/test"
"github.com/ipni/storetheindex/carstore"
"github.com/ipni/storetheindex/filestore"
crypto "github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
p2ptest "github.com/libp2p/go-libp2p/core/test"
"github.com/multiformats/go-multicodec"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -233,7 +232,7 @@ func newRandomLinkedList(t *testing.T, lsys ipld.LinkSystem, size int) (ipld.Lin
var out []multihash.Multihash
var nextLnk ipld.Link
for i := 0; i < size; i++ {
mhs := test.RandomMultihashes(testEntriesChunkSize)
mhs := random.Multihashes(testEntriesChunkSize)
chunk := &schema.EntryChunk{
Entries: mhs,
Next: nextLnk,
Expand Down Expand Up @@ -271,11 +270,7 @@ func mkProvLinkSystem(ds datastore.Datastore) ipld.LinkSystem {
func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevLink ipld.Link, dstore datastore.Datastore) (ipld.Link, *schema.Advertisement, []multihash.Multihash, peer.ID, crypto.PrivKey) {
lsys := mkProvLinkSystem(dstore)

priv, pubKey, err := p2ptest.RandTestKeyPair(crypto.Ed25519, 256)
require.NoError(t, err)

p, err := peer.IDFromPublicKey(pubKey)
require.NoError(t, err)
p, priv, _ := random.Identity()

ctxID := []byte("test-context-id")
if metadata == nil {
Expand All @@ -297,7 +292,7 @@ func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevL
adv.Entries, mhs = newRandomLinkedList(t, lsys, eChunkCount)
}

err = adv.Sign(priv)
err := adv.Sign(priv)
require.NoError(t, err)

node, err := adv.ToNode()
Expand Down
10 changes: 5 additions & 5 deletions command/synthetic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

agg "github.com/filecoin-project/go-dagaggregator-unixfs"
"github.com/ipfs/go-cid"
"github.com/ipni/go-libipni/test"
"github.com/ipfs/go-test/random"
"github.com/urfave/cli/v2"
)

Expand Down Expand Up @@ -131,7 +131,7 @@ func writeCidFile(fileName string, num int) error {
for curr < num {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
if _, err = w.WriteString(cids[i].String()); err != nil {
Expand Down Expand Up @@ -171,7 +171,7 @@ func writeCidFileOfSize(fileName string, size int) error {
for curr < size {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
c := cids[i]
Expand Down Expand Up @@ -212,7 +212,7 @@ func writeManifest(fileName string, num int) error {
for curr < num {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}

Expand Down Expand Up @@ -257,7 +257,7 @@ func writeManifestOfSize(fileName string, size int) error {
for curr < size {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
c := cids[i]
Expand Down
1 change: 0 additions & 1 deletion config/identity.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ type Identity struct {
}

func (i Identity) Decode() (peer.ID, ic.PrivKey, error) {

privKey, err := i.DecodePrivateKey("")
if err != nil {
return "", nil, fmt.Errorf("could not decode private key: %w", err)
Expand Down
Loading
Loading