Enable PANIC_ON_INVARIANT_VIOLATED for tests

scripts/docker-integration-tests/m3aggregator.Dockerfile

@@ -8,5 +8,7 @@ ADD ./m3aggregator.yml /etc/m3aggregator/m3aggregator.yml
 
 EXPOSE 6000-6001/tcp
 
+ENV PANIC_ON_INVARIANT_VIOLATED=true
+
 ENTRYPOINT [ "/bin/m3aggregator" ]
 CMD [ "-f", "/etc/m3aggregator/m3aggregator.yml" ]

scripts/docker-integration-tests/m3coordinator.Dockerfile

@@ -8,5 +8,7 @@ ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml
 
 EXPOSE 7201/tcp 7203/tcp
 
+ENV PANIC_ON_INVARIANT_VIOLATED=true
+
 ENTRYPOINT [ "/bin/m3coordinator" ]
 CMD [ "-f", "/etc/m3coordinator/m3coordinator.yml" ]

scripts/docker-integration-tests/m3dbnode.Dockerfile

@@ -8,5 +8,7 @@ ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml
 
 EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp
 
+ENV PANIC_ON_INVARIANT_VIOLATED=true
+
 ENTRYPOINT [ "/bin/m3dbnode" ]
 CMD [ "-f", "/etc/m3dbnode/m3dbnode.yml" ]

scripts/dtest/docker-compose.yml

@@ -10,6 +10,8 @@ services:
       - "0.0.0.0:9000:9000"
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
+    env:
+      - PANIC_ON_INVARIANT_VIOLATED=true
   coord01:
     networks:
       - dtest
@@ -20,5 +22,7 @@ services:
       - "0.0.0.0:7204:7204"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    env:
+      - PANIC_ON_INVARIANT_VIOLATED=true
 networks:
   dtest:

src/dbnode/client/session_aggregate_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/m3db/m3/src/m3ninx/idx"
 	xerrors "github.com/m3db/m3/src/x/errors"
 	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
 	xretry "github.com/m3db/m3/src/x/retry"
 	xtest "github.com/m3db/m3/src/x/test"
 
@@ -282,10 +283,11 @@ func TestSessionAggregateIDsEnqueueErr(t *testing.T) {
 
 	assert.NoError(t, session.Open())
 
-	_, _, err = session.Aggregate(testContext(), ident.StringID("namespace"),
-		testSessionAggregateQuery, testSessionAggregateQueryOpts(start, end))
-	assert.Error(t, err)
-	assert.NoError(t, session.Close())
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() {
+		_, _, _ = session.Aggregate(testContext(), ident.StringID("namespace"),
+			testSessionAggregateQuery, testSessionAggregateQueryOpts(start, end))
+	})
 }
 
 func TestSessionAggregateMergeTest(t *testing.T) {

src/dbnode/client/session_fetch_tagged_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/m3db/m3/src/m3ninx/idx"
 	xerrors "github.com/m3db/m3/src/x/errors"
 	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
 	xretry "github.com/m3db/m3/src/x/retry"
 	xtest "github.com/m3db/m3/src/x/test"
 
@@ -288,10 +289,11 @@ func TestSessionFetchTaggedIDsEnqueueErr(t *testing.T) {
 
 	assert.NoError(t, session.Open())
 
-	_, _, err = session.FetchTaggedIDs(testContext(), ident.StringID("namespace"),
-		testSessionFetchTaggedQuery, testSessionFetchTaggedQueryOpts(start, end))
-	assert.Error(t, err)
-	assert.NoError(t, session.Close())
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() {
+		_, _, _ = session.FetchTaggedIDs(testContext(), ident.StringID("namespace"),
+			testSessionFetchTaggedQuery, testSessionFetchTaggedQueryOpts(start, end))
+	})
 }
 
 func TestSessionFetchTaggedMergeTest(t *testing.T) {
@@ -588,6 +590,15 @@ func mockExtendedHostQueues(
 		require.Fail(t, "unable to find host idx: %v", host.ID())
 		return -1
 	}
+	expectClose := true
+	for _, hq := range opsByHost {
+		for _, e := range hq.enqueues {
+			if e.enqueueErr != nil {
+				expectClose = false
+				break
+			}
+		}
+	}
 	s.newHostQueueFn = func(
 		host topology.Host,
 		opts hostQueueOpts,
@@ -600,6 +611,7 @@ func mockExtendedHostQueues(
 		hostQueue.EXPECT().Open()
 		hostQueue.EXPECT().Host().Return(host).AnyTimes()
 		hostQueue.EXPECT().ConnectionCount().Return(opts.opts.MinConnectionCount()).AnyTimes()
+
 		var expectNextEnqueueFn func(fns []testEnqueue)
 		expectNextEnqueueFn = func(fns []testEnqueue) {
 			fn := fns[0]
@@ -619,7 +631,9 @@ func mockExtendedHostQueues(
 		if len(hostEnqueues.enqueues) > 0 {
 			expectNextEnqueueFn(hostEnqueues.enqueues)
 		}
-		hostQueue.EXPECT().Close()
+		if expectClose {
+			hostQueue.EXPECT().Close()
+		}
 		return hostQueue, nil
 	}
 }

src/dbnode/persist/fs/files_test.go

@@ -415,8 +415,10 @@ func TestFileExists(t *testing.T) {
 	require.NoError(t, err)
 	require.True(t, exists)
 
-	_, err = FileExists(checkpointFilePath)
-	require.Error(t, err)
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() {
+		_, _ = FileExists(checkpointFilePath)
+	})
 
 	os.Remove(infoFilePath)
 	require.False(t, mustFileExists(t, infoFilePath))
@@ -447,9 +449,8 @@ func TestCompleteCheckpointFileExists(t *testing.T) {
 	require.NoError(t, err)
 	require.True(t, exists)
 
-	exists, err = CompleteCheckpointFileExists("some-arbitrary-file")
-	require.Contains(t, err.Error(), instrument.InvariantViolatedMetricName)
-	require.False(t, exists)
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() { _, _ = CompleteCheckpointFileExists("some-arbitrary-file") })
 }
 
 func TestShardDirPath(t *testing.T) {

src/dbnode/persist/fs/index_claims_manager_test.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/m3db/m3/src/dbnode/namespace"
 	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
 	xtime "github.com/m3db/m3/src/x/time"
 )
 
@@ -49,9 +50,8 @@ func TestIndexClaimsManagerSingleGlobalManager(t *testing.T) {
 	require.NoError(t, err)
 
 	// Second should cause an error.
-	_, err = NewIndexClaimsManager(testDefaultOpts)
-	require.Error(t, err)
-	require.Equal(t, errMustUseSingleClaimsManager, err)
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() { _, _ = NewIndexClaimsManager(testDefaultOpts) })
 }
 
 func TestIndexClaimsManagerConcurrentClaims(t *testing.T) {

src/dbnode/persist/fs/persist_manager_test.go

@@ -36,6 +36,7 @@ import (
 	m3ninxpersist "github.com/m3db/m3/src/m3ninx/persist"
 	"github.com/m3db/m3/src/x/checked"
 	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
 	m3test "github.com/m3db/m3/src/x/test"
 	xtest "github.com/m3db/m3/src/x/test"
 
@@ -72,10 +73,9 @@ func TestPersistenceManagerPrepareDataFileExistsNoDelete(t *testing.T) {
 		Shard:             shard,
 		BlockStart:        blockStart,
 	}
-	prepared, err := flush.PrepareData(prepareOpts)
-	require.Equal(t, errPersistManagerFileSetAlreadyExists, err)
-	require.Nil(t, prepared.Persist)
-	require.Nil(t, prepared.Close)
+
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() { _, _ = flush.PrepareData(prepareOpts) })
 }
 
 func TestPersistenceManagerPrepareDataFileExistsWithDelete(t *testing.T) {

src/dbnode/storage/fs_test.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+
+	"github.com/m3db/m3/src/x/instrument"
 )
 
 func TestFileSystemManagerShouldRunDuringBootstrap(t *testing.T) {
@@ -86,9 +88,8 @@ func TestFileSystemManagerRun(t *testing.T) {
 	ts := time.Now()
 	gomock.InOrder(
 		cm.EXPECT().WarmFlushCleanup(ts, true).Return(errors.New("foo")),
-		fm.EXPECT().Flush(ts).Return(errors.New("bar")),
 	)
 
-	mgr.Run(ts, syncRun, noForce)
-	require.Equal(t, fileOpNotStarted, mgr.status)
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() { mgr.Run(ts, syncRun, noForce) })
 }

src/dbnode/storage/index_block_test.go

@@ -40,6 +40,7 @@ import (
 	"github.com/m3db/m3/src/x/context"
 	xerrors "github.com/m3db/m3/src/x/errors"
 	"github.com/m3db/m3/src/x/ident"
+	"github.com/m3db/m3/src/x/instrument"
 	xtest "github.com/m3db/m3/src/x/test"
 	xtime "github.com/m3db/m3/src/x/time"
 
@@ -196,11 +197,13 @@ func TestNamespaceIndexNewBlockFnRandomErr(t *testing.T) {
 	) (index.Block, error) {
 		return nil, fmt.Errorf("randomerr")
 	}
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
 	md := testNamespaceMetadata(blockSize, 4*time.Hour)
-	_, err := newNamespaceIndexWithNewBlockFn(md,
-		namespace.NewRuntimeOptionsManager(md.ID().String()),
-		testShardSet, newBlockFn, opts)
-	require.Error(t, err)
+	require.Panics(t, func() {
+		_, _ = newNamespaceIndexWithNewBlockFn(md,
+			namespace.NewRuntimeOptionsManager(md.ID().String()),
+			testShardSet, newBlockFn, opts)
+	})
 }
 
 func TestNamespaceIndexWrite(t *testing.T) {

src/dbnode/storage/namespace_test.go

@@ -490,6 +490,8 @@ func TestNamespaceBootstrapOnlyNonBootstrappedShards(t *testing.T) {
 	ctx := context.NewBackground()
 	defer ctx.Close()
 
+	// do not panic for invariant violation to test some shards are still bootstrapped.
+	defer instrument.SetShouldPanicEnvironmentVariable(false)()
 	require.Error(t, ns.Bootstrap(ctx, nsResult))
 	require.Equal(t, BootstrapNotStarted, ns.bootstrapState)
 }

src/query/storage/m3/storage_test.go

@@ -851,6 +851,6 @@ func TestInvalidBlockTypes(t *testing.T) {
 
 	query := &storage.FetchQuery{}
 	fetchOpts := &storage.FetchOptions{BlockType: models.TypeMultiBlock}
-	_, err = s.FetchBlocks(context.TODO(), query, fetchOpts)
-	assert.Error(t, err)
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
+	require.Panics(t, func() { _, _ = s.FetchBlocks(context.TODO(), query, fetchOpts) })
 }

src/x/instrument/invariant.go

@@ -23,6 +23,7 @@ package instrument
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"go.uber.org/zap"
@@ -88,6 +89,16 @@ func InvariantErrorf(format string, a ...interface{}) error {
 	return err
 }
 
+// SetShouldPanicEnvironmentVariable sets the env variable and returns a func to reset to the previous value.
+// Useful for tests to use a defer statement when they need to test a specific value.
+func SetShouldPanicEnvironmentVariable(value bool) func() {
+	restoreValue := os.Getenv(ShouldPanicEnvironmentVariableName)
+	_ = os.Setenv(ShouldPanicEnvironmentVariableName, strconv.FormatBool(value))
+	return func() {
+		_ = os.Setenv(ShouldPanicEnvironmentVariableName, restoreValue)
+	}
+}
+
 func panicIfEnvSet() {
 	panicIfEnvSetWithMessage("")
 }

src/x/instrument/invariant_test.go

@@ -22,7 +22,6 @@ package instrument_test
 
 import (
 	"fmt"
-	"os"
 	"testing"
 
 	"github.com/m3db/m3/src/x/instrument"
@@ -33,6 +32,7 @@ import (
 )
 
 func ExampleInvariantViolatedMetricInvocation() {
+	defer instrument.SetShouldPanicEnvironmentVariable(false)()
 	testScope := tally.NewTestScope("", nil)
 	opts := instrument.NewOptions().SetMetricsScope(testScope)
 
@@ -44,13 +44,15 @@ func ExampleInvariantViolatedMetricInvocation() {
 }
 
 func TestEmitInvariantViolationDoesNotPanicIfEnvNotSet(t *testing.T) {
+	defer instrument.SetShouldPanicEnvironmentVariable(false)()
 	opts := instrument.NewOptions()
 	require.NotPanics(t, func() {
 		instrument.EmitInvariantViolation(opts)
 	})
 }
 
 func TestEmitAndLogInvariantViolationDoesNotPanicIfEnvNotSet(t *testing.T) {
+	defer instrument.SetShouldPanicEnvironmentVariable(false)()
 	opts := instrument.NewOptions()
 	require.NotPanics(t, func() {
 		instrument.EmitAndLogInvariantViolation(opts, func(l *zap.Logger) {
@@ -60,6 +62,7 @@ func TestEmitAndLogInvariantViolationDoesNotPanicIfEnvNotSet(t *testing.T) {
 }
 
 func TestErrorfDoesNotPanicIfEnvNotSet(t *testing.T) {
+	defer instrument.SetShouldPanicEnvironmentVariable(false)()
 	var (
 		format      = "some error format: %s"
 		err_msg     = "error message"
@@ -72,7 +75,7 @@ func TestErrorfDoesNotPanicIfEnvNotSet(t *testing.T) {
 }
 
 func TestEmitInvariantViolationPanicsIfEnvSet(t *testing.T) {
-	defer setShouldPanicEnvironmentVariable()()
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
 
 	opts := instrument.NewOptions()
 	require.Panics(t, func() {
@@ -81,7 +84,7 @@ func TestEmitInvariantViolationPanicsIfEnvSet(t *testing.T) {
 }
 
 func TestEmitAndLogInvariantViolationPanicsIfEnvSet(t *testing.T) {
-	defer setShouldPanicEnvironmentVariable()()
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
 
 	opts := instrument.NewOptions()
 	require.Panics(t, func() {
@@ -92,18 +95,8 @@ func TestEmitAndLogInvariantViolationPanicsIfEnvSet(t *testing.T) {
 }
 
 func TestErrorfPanicsIfEnvSet(t *testing.T) {
-	defer setShouldPanicEnvironmentVariable()()
+	defer instrument.SetShouldPanicEnvironmentVariable(true)()
 	require.Panics(t, func() {
 		instrument.InvariantErrorf("some_error")
 	})
 }
-
-type cleanupFn func()
-
-func setShouldPanicEnvironmentVariable() cleanupFn {
-	restoreValue := os.Getenv(instrument.ShouldPanicEnvironmentVariableName)
-	os.Setenv(instrument.ShouldPanicEnvironmentVariableName, "true")
-	return cleanupFn(func() {
-		os.Setenv(instrument.ShouldPanicEnvironmentVariableName, restoreValue)
-	})
-}