Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(sims): Introduce new sims method factory and helper structures #21037

Closed
wants to merge 4 commits into from
Closed

refactor(sims): Introduce new sims method factory and helper structures #21037

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
690ec63
Introduce new sims method factory and helper structures
alpe Jul 17, 2024
3f891be
Address todos
alpe Jul 25, 2024
f495a50
Merge branch 'main' into alex/sims2_s1
alpe Jul 25, 2024
e7a9942
Merge branch 'main' into alex/sims2_s1
alpe Jul 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 14 additions & 41 deletions scripts/build/simulations.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#? test-sim-nondeterminism: Run non-determinism test for simapp
test-sim-nondeterminism:
@echo "Running non-determinism test..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
-NumBlocks=100 -BlockSize=200 -Period=0

# Requires an exported plugin. See store/streaming/README.md for documentation.
Expand All @@ -16,45 +16,45 @@ test-sim-nondeterminism:
# make test-sim-nondeterminism-streaming
test-sim-nondeterminism-streaming:
@echo "Running non-determinism-streaming test..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
-NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true

test-sim-custom-genesis-fast:
@echo "Running custom genesis simulation..."
@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false

test-sim-import-export:
@echo "Running application import/export simulation. This may take several minutes..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
-NumBlocks=50 -Period=5

test-sim-after-import:
@echo "Running application simulation-after-import. This may take several minutes..."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
-NumBlocks=50 -Period=5


test-sim-custom-genesis-multi-seed:
@echo "Running multi-seed custom genesis simulation..."
@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-NumBlocks=400 -Period=5

test-sim-multi-seed-long:
@echo "Running long multi-seed application simulation. This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -tags='sims' -run TestFullAppSimulation \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=1h -tags='sims' -run TestFullAppSimulation \
-NumBlocks=150 -Period=50

test-sim-multi-seed-short:
@echo "Running short multi-seed application simulation. This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
-NumBlocks=50 -Period=10
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
-NumBlocks=50 -Period=10 -FauxMerkle=true

test-sim-benchmark-invariants:
@echo "Running simulation invariant benchmarks..."
cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
-Enabled=true -NumBlocks=1000 -BlockSize=200 \
-Period=1 -Commit=true -Seed=57 -v -timeout 24h

Expand All @@ -77,50 +77,23 @@ SIM_COMMIT ?= true
test-sim-fuzz:
@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
#ld flags are a quick fix to make it work on current osx
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20

#? test-sim-benchmark: Run benchmark test for simapp
test-sim-benchmark:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
# export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
# make test-sim-benchmark-streaming
#
# Using the built-in examples:
# export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
# make test-sim-benchmark-streaming
test-sim-benchmark-streaming:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -EnableStreaming=true

test-sim-profile:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
@cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
# export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
# make test-sim-profile-streaming
#
# Using the built-in examples:
# export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
# make test-sim-profile-streaming
test-sim-profile-streaming:
@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true

.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz

#? benchmark: Run benchmark tests
benchmark:
@go test -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
@go test -failfast -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
.PHONY: benchmark
2 changes: 1 addition & 1 deletion simapp/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ func NewSimApp(
overrideModules := map[string]module.AppModuleSimulation{
authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AuthKeeper, app.AccountsKeeper, authsims.RandomGenesisAccounts),
}
app.sm = module.NewSimulationManagerFromAppModules(app.ModuleManager.Modules, overrideModules)
app.sm = module.NewSimulationManagerFromAppModules(app.AuthKeeper, app.BankKeeper, app.ModuleManager.Modules, overrideModules)

// create, start, and load the unordered tx manager
utxDataDir := filepath.Join(homePath, "data")
Expand Down
2 changes: 1 addition & 1 deletion simapp/app_di.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ func NewSimApp(
overrideModules := map[string]module.AppModuleSimulation{
authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AuthKeeper, &app.AccountsKeeper, authsims.RandomGenesisAccounts),
}
app.sm = module.NewSimulationManagerFromAppModules(app.ModuleManager.Modules, overrideModules)
app.sm = module.NewSimulationManagerFromAppModules(app.AuthKeeper, app.BankKeeper, app.ModuleManager.Modules, overrideModules)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

API breaking changelog entry in main changelog, and wiring change in simapp/CHANGELOG.md


app.sm.RegisterStoreDecoders()

Expand Down
68 changes: 1 addition & 67 deletions simapp/sim_bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,13 @@
package simapp

import (
"os"
"testing"

coretesting "cosmossdk.io/core/testing"
"github.com/cosmos/cosmos-sdk/testutils/sims"

flag "github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"

"github.com/cosmos/cosmos-sdk/baseapp"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/server"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
"github.com/cosmos/cosmos-sdk/x/simulation"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
)

var FlagEnableBenchStreamingValue bool

// Get flags every time the simulator is run
func init() {
flag.BoolVar(&FlagEnableBenchStreamingValue, "EnableStreaming", false, "Enable streaming service")
}

// Profile with:
// /usr/local/go/bin/go test -benchmem -run=^$ cosmossdk.io/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out
func BenchmarkFullAppSimulation(b *testing.B) {
Expand All @@ -37,52 +18,5 @@ func BenchmarkFullAppSimulation(b *testing.B) {
config := simcli.NewConfigFromFlags()
config.ChainID = sims.SimAppChainID

db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue)
if err != nil {
b.Fatalf("simulation setup failed: %s", err.Error())
}

if skip {
b.Skip("skipping benchmark application simulation")
}

defer func() {
require.NoError(b, db.Close())
require.NoError(b, os.RemoveAll(dir))
}()

appOptions := viper.New()
appOptions.SetDefault(flags.FlagHome, DefaultNodeHome)
appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue)

app := NewSimApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(sims.SimAppChainID))

// run randomized simulation
simParams, simErr := simulation.SimulateFromSeedX(
b,
coretesting.NewNopLogger(),
os.Stdout,
app.BaseApp,
simtestutil.AppStateFn(app.AppCodec(), app.AuthKeeper.AddressCodec(), app.StakingKeeper.ValidatorAddressCodec(), app.SimulationManager(), app.DefaultGenesis()),
simtypes.RandomAccounts,
simtestutil.SimulationOperations(app, app.AppCodec(), config, app.txConfig),
BlockedAddresses(),
config,
app.AppCodec(),
app.txConfig.SigningContext().AddressCodec(),
&simulation.DummyLogWriter{},
)

// export state and simParams before the simulation error is checked
if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil {
b.Fatal(err)
}

if simErr != nil {
b.Fatal(simErr)
}

if config.Commit {
simtestutil.PrintStats(db)
}
sims.RunWithSeed(b, config, NewSimApp, setupStateFactory, 1, nil)
}
59 changes: 27 additions & 32 deletions simapp/sim_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"
"sync"
"testing"
"time"

"cosmossdk.io/log"
"cosmossdk.io/store"
Expand Down Expand Up @@ -63,12 +64,12 @@ func setupStateFactory(app *SimApp) sims.SimStateFactory {
}

var (
exportAllModules = []string{}
exportWithValidatorSet = []string{}
exportAllModules []string
exportWithValidatorSet []string
)

func TestAppImportExport(t *testing.T) {
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
sims.Run(t, NewSimApp, setupStateFactory, func(t testing.TB, ti sims.TestInstance[*SimApp]) {
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
Expand Down Expand Up @@ -110,40 +111,34 @@ func TestAppImportExport(t *testing.T) {
// set up a new node instance, Init chain from exported genesis
// run new instance for n blocks
func TestAppSimulationAfterImport(t *testing.T) {
sims.Run(t, NewSimApp, setupStateFactory, func(t *testing.T, ti sims.TestInstance[*SimApp]) {
sims.Run(t, NewSimApp, setupStateFactory, func(t testing.TB, ti sims.TestInstance[*SimApp]) {
app := ti.App
t.Log("exporting genesis...\n")
exported, err := app.ExportAppStateAndValidators(false, exportWithValidatorSet, exportAllModules)
require.NoError(t, err)

t.Log("importing genesis...\n")
newTestInstance := sims.NewSimulationAppInstance(t, ti.Cfg, NewSimApp)
newApp := newTestInstance.App
_, err = newApp.InitChain(&abci.InitChainRequest{
AppStateBytes: exported.AppState,
ChainId: sims.SimAppChainID,
})
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return
importGenesisStateFactory := func(app *SimApp) sims.SimStateFactory {
return sims.SimStateFactory{
Codec: app.AppCodec(),
AppStateFn: func(r *rand.Rand, accs []simtypes.Account, config simtypes.Config) (json.RawMessage, []simtypes.Account, string, time.Time) {
_, err = app.InitChain(&abci.InitChainRequest{
AppStateBytes: exported.AppState,
ChainId: sims.SimAppChainID,
})
if IsEmptyValidatorSetErr(err) {
t.Skip("Skipping simulation as all validators have been unbonded")
return nil, nil, "", time.Time{}
}
acc, err := simtestutil.AccountsFromAppState(app.AppCodec(), exported.AppState)
require.NoError(t, err)
genesisTimestamp := time.Unix(config.GenesisTime, 0)
return exported.AppState, acc, config.ChainID, genesisTimestamp
},
BlockedAddr: BlockedAddresses(),
}
}
require.NoError(t, err)
newStateFactory := setupStateFactory(newApp)
_, err = simulation.SimulateFromSeedX(
t,
newTestInstance.AppLogger,
sims.WriteToDebugLog(newTestInstance.AppLogger),
newApp.BaseApp,
newStateFactory.AppStateFn,
simtypes.RandomAccounts,
simtestutil.SimulationOperations(newApp, newApp.AppCodec(), newTestInstance.Cfg, newApp.TxConfig()),
newStateFactory.BlockedAddr,
newTestInstance.Cfg,
newStateFactory.Codec,
newApp.TxConfig().SigningContext().AddressCodec(),
ti.ExecLogWriter,
)
require.NoError(t, err)
sims.RunWithSeed(t, ti.Cfg, NewSimApp, importGenesisStateFactory, ti.Cfg.Seed, ti.Cfg.FuzzSeed)
})
}

Expand Down Expand Up @@ -189,7 +184,7 @@ func TestAppStateDeterminism(t *testing.T) {
var mx sync.Mutex
appHashResults := make(map[int64][][]byte)
appSimLogger := make(map[int64][]simulation.LogWriter)
captureAndCheckHash := func(t *testing.T, ti sims.TestInstance[*SimApp]) {
captureAndCheckHash := func(t testing.TB, ti sims.TestInstance[*SimApp]) {
seed, appHash := ti.Cfg.Seed, ti.App.LastCommitID().Hash
mx.Lock()
otherHashes, execWriters := appHashResults[seed], appSimLogger[seed]
Expand Down Expand Up @@ -225,7 +220,7 @@ type ComparableStoreApp interface {
GetStoreKeys() []storetypes.StoreKey
}

func AssertEqualStores(t *testing.T, app ComparableStoreApp, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) {
func AssertEqualStores(t testing.TB, app ComparableStoreApp, newApp ComparableStoreApp, storeDecoders simtypes.StoreDecoderRegistry, skipPrefixes map[string][][]byte) {
ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})
ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()})

Expand Down
Loading
Loading