diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 4a3e3cebfa8..6ee51e516fa 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -7,10 +7,10 @@ import ( "time" humanize "github.com/dustin/go-humanize" + "github.com/ipfs/boxo/provider" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" - - "github.com/ipfs/boxo/provider" + "golang.org/x/exp/constraints" ) var statProvideCmd = &cmds.Command{ @@ -64,7 +64,7 @@ func humanDuration(val time.Duration) string { return val.Truncate(time.Microsecond).String() } -func humanNumber(n int) string { +func humanNumber[T constraints.Float | constraints.Integer](n T) string { nf := float64(n) str := humanSI(nf, 0) fullStr := humanFull(nf, 0) diff --git a/core/node/provider.go b/core/node/provider.go index 387815db980..a97a96f82e5 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/fetcher" pin "github.com/ipfs/boxo/pinning/pinner" provider "github.com/ipfs/boxo/provider" @@ -14,11 +15,87 @@ import ( ) func ProviderSys(reprovideInterval time.Duration) fx.Option { - return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo) (provider.System, error) { + const magicThroughputReportCount = 128 + return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { sys, err := provider.New(repo.Datastore(), provider.Online(cr), provider.ReproviderInterval(reprovideInterval), provider.KeyProvider(keyProvider), + provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool { + avgProvideSpeed := duration / time.Duration(keysProvided) + count := uint64(keysProvided) + + if !reprovide || !complete { + // We don't know how many CIDs we have to provide, try to fetch it from the blockstore. + // But don't try for too long as this might be very expensive if you have a huge datastore. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + // FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup. + ch, err := bs.AllKeysChan(ctx) + if err != nil { + logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err) + return false + } + count = 0 + countLoop: + for { + select { + case _, ok := <-ch: + if !ok { + break countLoop + } + count++ + case <-ctx.Done(): + // really big blockstore mode + + // how many blocks would be in a 10TiB blockstore with 128KiB blocks. + const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024) + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / probableBigBlockstore + if avgProvideSpeed > expectedProvideSpeed { + // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore. + logger.Errorf(` +🔔🔔🔔 YOU MAY BE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔 + +⚠️ Your system might be struggling to keep up with DHT reprovides! +Meaning your content being partially or completely inacessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +🕑 An attempt to estimate your blockstore size timed out after 5 minutes, +implying your blockstore might be exceedingly large. Assuming a considerable +size of 10TiB, it would take %v to provide the complete set. + +⏰ The total provide time needs to stay under 24 hours to prevent falling behind! + +💡 Consider enabling the Accelerated DHT to enhance your system performance.`, + keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore) + return false + } + } + } + } + + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / time.Duration(count) + if avgProvideSpeed > expectedProvideSpeed { + // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore. + logger.Errorf(` +🔔🔔🔔 YOU ARE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔 + +⚠️ Your system is struggling to keep up with DHT reprovides! +Meaning your content being partially or completely inacessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +💾 Your total CID count is ~%d which would total at %v reprovide process. + +⏰ The total provide time needs to stay under 24 hours to prevent falling behind! + +💡 Consider enabling the Accelerated DHT to enhance your reprovide throughput.`, + keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count)) + } + return false + }, magicThroughputReportCount), ) if err != nil { return nil, err @@ -56,7 +133,7 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro return fx.Options( keyProvider, - fx.Provide(ProviderSys(reprovideInterval)), + ProviderSys(reprovideInterval), ) } diff --git a/go.mod b/go.mod index 195f2a23aea..e4bda7daa4d 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b + github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -81,6 +81,7 @@ require ( go.uber.org/fx v1.19.2 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.9.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/mod v0.10.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.8.0 @@ -214,7 +215,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/term v0.8.0 // indirect diff --git a/go.sum b/go.sum index fe12a3d47e7..b6b46630317 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230511175036-cedadbf667d7 h1:Nbe1zkYEMv81oRfJJH2kCyXr+ENuR+mbTko7/jvXEs8= +github.com/ipfs/boxo v0.8.2-0.20230511175036-cedadbf667d7/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= +github.com/ipfs/boxo v0.8.2-0.20230511183253-30d1ef9a3fa6 h1:qaocp22F74W6YHDEE4qxS5Zn2b6lDN5mZeBbIVAgqFA= +github.com/ipfs/boxo v0.8.2-0.20230511183253-30d1ef9a3fa6/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= +github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd h1:2WrEH+ymhrJK7G0ouf6GhUvFTY7fBr0labD4+DHmgok= +github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=