feat: warn users who are falling behind reprovides

Fixes: ipfs#9704
Fixes: ipfs#9702
Depends on: ipfs#273

core/commands/stat_provide.go

@@ -7,10 +7,10 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/ipfs/boxo/provider"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
-
-	"github.com/ipfs/boxo/provider"
+	"golang.org/x/exp/constraints"
 )
 
 var statProvideCmd = &cmds.Command{
@@ -64,7 +64,7 @@ func humanDuration(val time.Duration) string {
 	return val.Truncate(time.Microsecond).String()
 }
 
-func humanNumber(n int) string {
+func humanNumber[T constraints.Float | constraints.Integer](n T) string {
 	nf := float64(n)
 	str := humanSI(nf, 0)
 	fullStr := humanFull(nf, 0)

core/node/provider.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/ipfs/boxo/blockstore"
 	"github.com/ipfs/boxo/fetcher"
 	pin "github.com/ipfs/boxo/pinning/pinner"
 	provider "github.com/ipfs/boxo/provider"
@@ -14,11 +15,87 @@ import (
 )
 
 func ProviderSys(reprovideInterval time.Duration) fx.Option {
-	return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo) (provider.System, error) {
+	const magicThroughputReportCount = 128
+	return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) {
 		sys, err := provider.New(repo.Datastore(),
 			provider.Online(cr),
 			provider.ReproviderInterval(reprovideInterval),
 			provider.KeyProvider(keyProvider),
+			provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool {
+				avgProvideSpeed := duration / time.Duration(keysProvided)
+				count := uint64(keysProvided)
+
+				if !reprovide || !complete {
+					// We don't know how many CIDs we have to provide, try to fetch it from the blockstore.
+					// But don't try for too long as this might be very expensive if you have a huge datastore.
+					ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+					defer cancel()
+
+					// FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup.
+					ch, err := bs.AllKeysChan(ctx)
+					if err != nil {
+						logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err)
+						return false
+					}
+					count = 0
+				countLoop:
+					for {
+						select {
+						case _, ok := <-ch:
+							if !ok {
+								break countLoop
+							}
+							count++
+						case <-ctx.Done():
+							// really big blockstore mode
+
+							// how many blocks would be in a 10TiB blockstore with 128KiB blocks.
+							const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024)
+							// How long per block that lasts us.
+							expectedProvideSpeed := reprovideInterval / probableBigBlockstore
+							if avgProvideSpeed > expectedProvideSpeed {
+								// FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore.
+								logger.Errorf(`
+🔔🔔🔔 YOU MAY BE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
+
+⚠️ Your system might be struggling to keep up with DHT reprovides!
+Meaning your content being partially or completely inacessible on the network.
+We observed that you recently provided %d keys at an average rate of %v per key.
+
+🕑 An attempt to estimate your blockstore size timed out after 5 minutes,
+implying your blockstore might be exceedingly large. Assuming a considerable
+size of 10TiB, it would take %v to provide the complete set.
+
+⏰ The total provide time needs to stay under 24 hours to prevent falling behind!
+
+💡 Consider enabling the Accelerated DHT to enhance your system performance.`,
+									keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore)
+								return false
+							}
+						}
+					}
+				}
+
+				// How long per block that lasts us.
+				expectedProvideSpeed := reprovideInterval / time.Duration(count)
+				if avgProvideSpeed > expectedProvideSpeed {
+					// FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore.
+					logger.Errorf(`
+🔔🔔🔔 YOU ARE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
+
+⚠️ Your system is struggling to keep up with DHT reprovides!
+Meaning your content being partially or completely inacessible on the network.
+We observed that you recently provided %d keys at an average rate of %v per key.
+
+💾 Your total CID count is ~%d which would total at %v reprovide process.
+
+⏰ The total provide time needs to stay under 24 hours to prevent falling behind!
+
+💡 Consider enabling the Accelerated DHT to enhance your reprovide throughput.`,
+						keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count))
+				}
+				return false
+			}, magicThroughputReportCount),
 		)
 		if err != nil {
 			return nil, err
@@ -56,7 +133,7 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
 
 	return fx.Options(
 		keyProvider,
-		fx.Provide(ProviderSys(reprovideInterval)),
+		ProviderSys(reprovideInterval),
 	)
 }
 

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b
+	github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0
@@ -81,6 +81,7 @@ require (
 	go.uber.org/fx v1.19.2
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.9.0
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/mod v0.10.0
 	golang.org/x/sync v0.1.0
 	golang.org/x/sys v0.8.0
@@ -214,7 +215,6 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/term v0.8.0 // indirect

go.sum

@@ -354,8 +354,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc=
-github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
+github.com/ipfs/boxo v0.8.2-0.20230511175036-cedadbf667d7 h1:Nbe1zkYEMv81oRfJJH2kCyXr+ENuR+mbTko7/jvXEs8=
+github.com/ipfs/boxo v0.8.2-0.20230511175036-cedadbf667d7/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc=
+github.com/ipfs/boxo v0.8.2-0.20230511183253-30d1ef9a3fa6 h1:qaocp22F74W6YHDEE4qxS5Zn2b6lDN5mZeBbIVAgqFA=
+github.com/ipfs/boxo v0.8.2-0.20230511183253-30d1ef9a3fa6/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc=
+github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd h1:2WrEH+ymhrJK7G0ouf6GhUvFTY7fBr0labD4+DHmgok=
+github.com/ipfs/boxo v0.8.2-0.20230515125142-da153f4690bd/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=