From 938782685af444855e4f0044d17cb26164bf58fa Mon Sep 17 00:00:00 2001 From: Gediminas Guoba Date: Thu, 21 Jan 2021 10:22:50 +0200 Subject: [PATCH] [tests] test setups exported to allow us to use it from other packages (#3042) --- .../integration/cluster_add_one_node_test.go | 12 ++-- .../commitlog_bootstrap_unowned_shard_test.go | 8 +-- src/dbnode/integration/integration.go | 58 +++++++++++-------- src/dbnode/integration/options.go | 17 ++++++ .../peers_bootstrap_high_concurrency_test.go | 12 ++-- .../peers_bootstrap_index_aggregate_test.go | 8 +-- .../integration/peers_bootstrap_index_test.go | 8 +-- .../peers_bootstrap_merge_local_test.go | 14 ++--- .../peers_bootstrap_merge_peer_blocks_test.go | 10 ++-- .../peers_bootstrap_node_down_test.go | 10 ++-- .../peers_bootstrap_none_available_test.go | 12 ++-- .../peers_bootstrap_select_best_test.go | 10 ++-- .../peers_bootstrap_simple_test.go | 8 +-- .../peers_bootstrap_single_node_test.go | 6 +- src/dbnode/integration/repair_test.go | 10 ++-- src/dbnode/integration/setup.go | 24 +++++--- 16 files changed, 131 insertions(+), 96 deletions(-) diff --git a/src/dbnode/integration/cluster_add_one_node_test.go b/src/dbnode/integration/cluster_add_one_node_test.go index d2b4c1beb9..1d04c92d25 100644 --- a/src/dbnode/integration/cluster_add_one_node_test.go +++ b/src/dbnode/integration/cluster_add_one_node_test.go @@ -112,17 +112,17 @@ func testClusterAddOneNode(t *testing.T, verifyCommitlogCanBootstrapAfterNodeJoi topoOpts := topology.NewDynamicOptions(). SetConfigServiceClient(fake.NewM3ClusterClient(svcs, nil)) topoInit := topology.NewDynamicInitializer(topoOpts) - setupOpts := []bootstrappableTestSetupOptions{ + setupOpts := []BootstrappableTestSetupOptions{ { - disablePeersBootstrapper: true, - topologyInitializer: topoInit, + DisablePeersBootstrapper: true, + TopologyInitializer: topoInit, }, { - disablePeersBootstrapper: false, - topologyInitializer: topoInit, + DisablePeersBootstrapper: false, + TopologyInitializer: topoInit, }, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node. diff --git a/src/dbnode/integration/commitlog_bootstrap_unowned_shard_test.go b/src/dbnode/integration/commitlog_bootstrap_unowned_shard_test.go index 74c4b19162..fc062f5b26 100644 --- a/src/dbnode/integration/commitlog_bootstrap_unowned_shard_test.go +++ b/src/dbnode/integration/commitlog_bootstrap_unowned_shard_test.go @@ -89,12 +89,12 @@ func TestCommitLogBootstrapUnownedShard(t *testing.T) { opts := NewTestOptions(t). SetNamespaces([]namespace.Metadata{ns1}). SetNumShards(numShards) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true, topologyInitializer: topoInit}, - {disablePeersBootstrapper: true, topologyInitializer: topoInit}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true, TopologyInitializer: topoInit}, + {DisablePeersBootstrapper: true, TopologyInitializer: topoInit}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Only set this up for the first setup because we're only writing commit diff --git a/src/dbnode/integration/integration.go b/src/dbnode/integration/integration.go index 1aa2d56d13..cdd307dbb1 100644 --- a/src/dbnode/integration/integration.go +++ b/src/dbnode/integration/integration.go @@ -93,6 +93,7 @@ func newMultiAddrAdminClient( topologyInitializer topology.Initializer, origin topology.Host, instrumentOpts instrument.Options, + customOpts ...client.CustomAdminOption, ) client.AdminClient { if adminOpts == nil { adminOpts = client.NewAdminOptions() @@ -105,22 +106,28 @@ func newMultiAddrAdminClient( SetTopologyInitializer(topologyInitializer). SetClusterConnectTimeout(time.Second).(client.AdminOptions) + for _, o := range customOpts { + adminOpts = o(adminOpts) + } + adminClient, err := client.NewAdminClient(adminOpts) require.NoError(t, err) return adminClient } -type bootstrappableTestSetupOptions struct { - finalBootstrapper string - bootstrapBlocksBatchSize int - bootstrapBlocksConcurrency int - bootstrapConsistencyLevel topology.ReadConsistencyLevel - topologyInitializer topology.Initializer - testStatsReporter xmetrics.TestStatsReporter - disablePeersBootstrapper bool - useTChannelClientForWriting bool - enableRepairs bool +// BootstrappableTestSetupOptions defines options for test setups. +type BootstrappableTestSetupOptions struct { + FinalBootstrapper string + BootstrapBlocksBatchSize int + BootstrapBlocksConcurrency int + BootstrapConsistencyLevel topology.ReadConsistencyLevel + TopologyInitializer topology.Initializer + TestStatsReporter xmetrics.TestStatsReporter + DisablePeersBootstrapper bool + UseTChannelClientForWriting bool + EnableRepairs bool + AdminClientCustomOpts []client.CustomAdminOption } type closeFn func() @@ -135,10 +142,11 @@ func newDefaulTestResultOptions( SetSeriesCachePolicy(storageOpts.SeriesCachePolicy()) } -func newDefaultBootstrappableTestSetups( +// NewDefaultBootstrappableTestSetups creates dbnode test setups. +func NewDefaultBootstrappableTestSetups( // nolint:gocyclo t *testing.T, opts TestOptions, - setupOpts []bootstrappableTestSetupOptions, + setupOpts []BootstrappableTestSetupOptions, ) (testSetups, closeFn) { var ( replicas = len(setupOpts) @@ -158,17 +166,18 @@ func newDefaultBootstrappableTestSetups( for i := 0; i < replicas; i++ { var ( instance = i - usingPeersBootstrapper = !setupOpts[i].disablePeersBootstrapper - finalBootstrapperToUse = setupOpts[i].finalBootstrapper - useTChannelClientForWriting = setupOpts[i].useTChannelClientForWriting - bootstrapBlocksBatchSize = setupOpts[i].bootstrapBlocksBatchSize - bootstrapBlocksConcurrency = setupOpts[i].bootstrapBlocksConcurrency - bootstrapConsistencyLevel = setupOpts[i].bootstrapConsistencyLevel - topologyInitializer = setupOpts[i].topologyInitializer - testStatsReporter = setupOpts[i].testStatsReporter - enableRepairs = setupOpts[i].enableRepairs + usingPeersBootstrapper = !setupOpts[i].DisablePeersBootstrapper + finalBootstrapperToUse = setupOpts[i].FinalBootstrapper + useTChannelClientForWriting = setupOpts[i].UseTChannelClientForWriting + bootstrapBlocksBatchSize = setupOpts[i].BootstrapBlocksBatchSize + bootstrapBlocksConcurrency = setupOpts[i].BootstrapBlocksConcurrency + bootstrapConsistencyLevel = setupOpts[i].BootstrapConsistencyLevel + topologyInitializer = setupOpts[i].TopologyInitializer + testStatsReporter = setupOpts[i].TestStatsReporter + enableRepairs = setupOpts[i].EnableRepairs origin topology.Host instanceOpts = newMultiAddrTestOptions(opts, instance) + adminClientCustomOpts = setupOpts[i].AdminClientCustomOpts ) if finalBootstrapperToUse == "" { @@ -211,7 +220,7 @@ func newDefaultBootstrappableTestSetups( // claim manager instances after the initial node. persistfs.ResetIndexClaimsManagersUnsafe() } - setup, err := NewTestSetup(t, instanceOpts, nil) + setup, err := NewTestSetup(t, instanceOpts, nil, opts.StorageOptsFn()) require.NoError(t, err) topologyInitializer = setup.TopologyInitializer() @@ -260,8 +269,11 @@ func newDefaultBootstrappableTestSetups( adminOpts = adminOpts.SetFetchSeriesBlocksBatchConcurrency(bootstrapBlocksConcurrency) } adminOpts = adminOpts.SetStreamBlocksRetrier(retrier) + adminClient := newMultiAddrAdminClient( - t, adminOpts, topologyInitializer, origin, instrumentOpts) + t, adminOpts, topologyInitializer, origin, instrumentOpts, adminClientCustomOpts...) + setup.SetStorageOpts(setup.StorageOpts().SetAdminClient(adminClient)) + storageIdxOpts := setup.StorageOpts().IndexOptions() fsOpts := setup.StorageOpts().CommitLogOptions().FilesystemOptions() if usingPeersBootstrapper { diff --git a/src/dbnode/integration/options.go b/src/dbnode/integration/options.go index 355b745c59..f2432bd0b4 100644 --- a/src/dbnode/integration/options.go +++ b/src/dbnode/integration/options.go @@ -293,6 +293,12 @@ type TestOptions interface { // ReportInterval returns the time between reporting metrics within the system. ReportInterval() time.Duration + + // SetStorageOptsFn sets the StorageOpts modifier. + SetStorageOptsFn(StorageOption) TestOptions + + // StorageOptsFn returns the StorageOpts modifier. + StorageOptsFn() StorageOption } type options struct { @@ -327,6 +333,7 @@ type options struct { assertEqual assertTestDataEqual nowFn func() time.Time reportInterval time.Duration + storageOptsFn StorageOption } // NewTestOptions returns a new set of integration test options. @@ -676,3 +683,13 @@ func (o *options) SetReportInterval(value time.Duration) TestOptions { func (o *options) ReportInterval() time.Duration { return o.reportInterval } + +func (o *options) SetStorageOptsFn(storageOptsFn StorageOption) TestOptions { + opts := *o + opts.storageOptsFn = storageOptsFn + return &opts +} + +func (o *options) StorageOptsFn() StorageOption { + return o.storageOptsFn +} diff --git a/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go b/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go index 0394153c95..98e9ffca8e 100644 --- a/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go +++ b/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go @@ -102,17 +102,17 @@ func testPeersBootstrapHighConcurrency( batchSize := 16 concurrency := 64 - setupOpts := []bootstrappableTestSetupOptions{ + setupOpts := []BootstrappableTestSetupOptions{ { - disablePeersBootstrapper: true, + DisablePeersBootstrapper: true, }, { - disablePeersBootstrapper: false, - bootstrapBlocksBatchSize: batchSize, - bootstrapBlocksConcurrency: concurrency, + DisablePeersBootstrapper: false, + BootstrapBlocksBatchSize: batchSize, + BootstrapBlocksConcurrency: concurrency, }, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node diff --git a/src/dbnode/integration/peers_bootstrap_index_aggregate_test.go b/src/dbnode/integration/peers_bootstrap_index_aggregate_test.go index 10a2d2890b..498fc538d6 100644 --- a/src/dbnode/integration/peers_bootstrap_index_aggregate_test.go +++ b/src/dbnode/integration/peers_bootstrap_index_aggregate_test.go @@ -65,11 +65,11 @@ func TestPeersBootstrapIndexAggregateQuery(t *testing.T) { SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node diff --git a/src/dbnode/integration/peers_bootstrap_index_test.go b/src/dbnode/integration/peers_bootstrap_index_test.go index d5907d9dc1..5b5e58761e 100644 --- a/src/dbnode/integration/peers_bootstrap_index_test.go +++ b/src/dbnode/integration/peers_bootstrap_index_test.go @@ -70,11 +70,11 @@ func TestPeersBootstrapIndexWithIndexingEnabled(t *testing.T) { SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node diff --git a/src/dbnode/integration/peers_bootstrap_merge_local_test.go b/src/dbnode/integration/peers_bootstrap_merge_local_test.go index b6638d7476..845ab95497 100644 --- a/src/dbnode/integration/peers_bootstrap_merge_local_test.go +++ b/src/dbnode/integration/peers_bootstrap_merge_local_test.go @@ -73,15 +73,15 @@ func testPeersBootstrapMergeLocal(t *testing.T, setTestOpts setTestOptions, upda // Enable useTchannelClientForWriting because this test relies upon being // able to write data to a single node, and the M3DB client does not support // that, but we can accomplish it by using an individual nodes TChannel endpoints. - setupOpts = []bootstrappableTestSetupOptions{ + setupOpts = []BootstrappableTestSetupOptions{ { - disablePeersBootstrapper: true, - useTChannelClientForWriting: true, + DisablePeersBootstrapper: true, + UseTChannelClientForWriting: true, }, { - disablePeersBootstrapper: false, - useTChannelClientForWriting: true, - testStatsReporter: reporter, + DisablePeersBootstrapper: false, + UseTChannelClientForWriting: true, + TestStatsReporter: reporter, }, } ) @@ -91,7 +91,7 @@ func testPeersBootstrapMergeLocal(t *testing.T, setTestOpts setTestOptions, upda namesp = opts.Namespaces()[0] } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node, ensure to overflow past diff --git a/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go b/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go index 40f1daca66..52e671f958 100644 --- a/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go +++ b/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go @@ -69,12 +69,12 @@ func testPeersBootstrapMergePeerBlocks(t *testing.T, setTestOpts setTestOptions, opts = setTestOpts(t, opts) namesp = opts.Namespaces()[0] } - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data alternating missing data for left/right nodes diff --git a/src/dbnode/integration/peers_bootstrap_node_down_test.go b/src/dbnode/integration/peers_bootstrap_node_down_test.go index 5518b1bb44..45c0f3753f 100644 --- a/src/dbnode/integration/peers_bootstrap_node_down_test.go +++ b/src/dbnode/integration/peers_bootstrap_node_down_test.go @@ -57,12 +57,12 @@ func TestPeersBootstrapNodeDown(t *testing.T) { SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node diff --git a/src/dbnode/integration/peers_bootstrap_none_available_test.go b/src/dbnode/integration/peers_bootstrap_none_available_test.go index b4d929c391..17a41c4c49 100644 --- a/src/dbnode/integration/peers_bootstrap_none_available_test.go +++ b/src/dbnode/integration/peers_bootstrap_none_available_test.go @@ -89,17 +89,17 @@ func TestPeersBootstrapNoneAvailable(t *testing.T) { SetShardSet(shardSet) topoInit := topology.NewStaticInitializer(topoOpts) - setupOpts := []bootstrappableTestSetupOptions{ + setupOpts := []BootstrappableTestSetupOptions{ { - disablePeersBootstrapper: false, - topologyInitializer: topoInit, + DisablePeersBootstrapper: false, + TopologyInitializer: topoInit, }, { - disablePeersBootstrapper: false, - topologyInitializer: topoInit, + DisablePeersBootstrapper: false, + TopologyInitializer: topoInit, }, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() serversAreUp := &sync.WaitGroup{} diff --git a/src/dbnode/integration/peers_bootstrap_select_best_test.go b/src/dbnode/integration/peers_bootstrap_select_best_test.go index 62efd41497..58caec15f2 100644 --- a/src/dbnode/integration/peers_bootstrap_select_best_test.go +++ b/src/dbnode/integration/peers_bootstrap_select_best_test.go @@ -56,12 +56,12 @@ func TestPeersBootstrapSelectBest(t *testing.T) { SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data alternating missing data for left/right nodes diff --git a/src/dbnode/integration/peers_bootstrap_simple_test.go b/src/dbnode/integration/peers_bootstrap_simple_test.go index 620bdc0b15..38bba77f9c 100644 --- a/src/dbnode/integration/peers_bootstrap_simple_test.go +++ b/src/dbnode/integration/peers_bootstrap_simple_test.go @@ -67,11 +67,11 @@ func testPeersBootstrapSimple(t *testing.T, setTestOpts setTestOptions, updateIn namesp = opts.Namespaces()[0] } - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true}, + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data for first node diff --git a/src/dbnode/integration/peers_bootstrap_single_node_test.go b/src/dbnode/integration/peers_bootstrap_single_node_test.go index 288d3c293c..7e48530f8c 100644 --- a/src/dbnode/integration/peers_bootstrap_single_node_test.go +++ b/src/dbnode/integration/peers_bootstrap_single_node_test.go @@ -57,10 +57,10 @@ func TestPeersBootstrapSingleNode(t *testing.T) { SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: false}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: false}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Write test data diff --git a/src/dbnode/integration/repair_test.go b/src/dbnode/integration/repair_test.go index b3a93e7bca..d469773777 100644 --- a/src/dbnode/integration/repair_test.go +++ b/src/dbnode/integration/repair_test.go @@ -196,12 +196,12 @@ func testRepair( SetUseTChannelClientForWriting(true). SetUseTChannelClientForReading(true) - setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: true, enableRepairs: true}, - {disablePeersBootstrapper: true, enableRepairs: true}, - {disablePeersBootstrapper: true, enableRepairs: true}, + setupOpts := []BootstrappableTestSetupOptions{ + {DisablePeersBootstrapper: true, EnableRepairs: true}, + {DisablePeersBootstrapper: true, EnableRepairs: true}, + {DisablePeersBootstrapper: true, EnableRepairs: true}, } - setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) + setups, closeFn := NewDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() // Ensure that the current time is set such that the previous block is flushable. diff --git a/src/dbnode/integration/setup.go b/src/dbnode/integration/setup.go index e4d27478cf..3c4c05ce6c 100644 --- a/src/dbnode/integration/setup.go +++ b/src/dbnode/integration/setup.go @@ -186,14 +186,15 @@ type TestSetup interface { InitializeBootstrappers(opts InitializeBootstrappersOptions) error } -type storageOption func(storage.Options) storage.Options +// StorageOption is a reference to storage options function. +type StorageOption func(storage.Options) storage.Options // NewTestSetup returns a new test setup for non-dockerized integration tests. func NewTestSetup( t *testing.T, opts TestOptions, fsOpts fs.Options, - storageOptFns ...storageOption, + storageOptFns ...StorageOption, ) (TestSetup, error) { if opts == nil { opts = NewTestOptions(t) @@ -465,7 +466,12 @@ func NewTestSetup( } for _, fn := range storageOptFns { - storageOpts = fn(storageOpts) + if fn != nil { + storageOpts = fn(storageOpts) + } + } + if storageOpts != nil && storageOpts.AdminClient() == nil { + storageOpts = storageOpts.SetAdminClient(adminClient) } return &testSetup{ @@ -1098,18 +1104,18 @@ func newNodes( SetConfigServiceClient(fake.NewM3ClusterClient(svcs, nil)) topoInit := topology.NewDynamicInitializer(topoOpts) - nodeOpt := bootstrappableTestSetupOptions{ - disablePeersBootstrapper: true, - finalBootstrapper: bootstrapper.NoOpAllBootstrapperName, - topologyInitializer: topoInit, + nodeOpt := BootstrappableTestSetupOptions{ + DisablePeersBootstrapper: true, + FinalBootstrapper: bootstrapper.NoOpAllBootstrapperName, + TopologyInitializer: topoInit, } - nodeOpts := make([]bootstrappableTestSetupOptions, len(instances)) + nodeOpts := make([]BootstrappableTestSetupOptions, len(instances)) for i := range instances { nodeOpts[i] = nodeOpt } - nodes, closeFn := newDefaultBootstrappableTestSetups(t, opts, nodeOpts) + nodes, closeFn := NewDefaultBootstrappableTestSetups(t, opts, nodeOpts) nodeClose := func() { // Clean up running servers at end of test log.Debug("servers closing")