Merge pull request #25 from ipfs/nopfs-subscriptions

Subscribe to badbits by default
ipfs · Oct 16, 2023 · 547f6fe · 547f6fe
2 parents f4bc5b2 + 3ecc869
commit 547f6fe
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 18 deletions.
diff --git a/go.mod b/go.mod
@@ -5,8 +5,8 @@ go 1.20
 require (
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/dustin/go-humanize v1.0.0
-	github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941
-	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71
+	github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3
+	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3
 	github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0

diff --git a/go.sum b/go.sum
@@ -287,10 +287,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
 github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 h1:IkOu07ym9Pi+O5Xl50CpMjvqULLFatzNY3+dyI54swM=
-github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg=
-github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 h1:gMcb3mZGIzSngq4oGI2L/j0X0kGqvEuaNm2dpqPTtbU=
-github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3 h1:eB1izctgB0japd/TG9hGMblUlwF82lwrTo64cwNl5Po=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3 h1:4sJsKDnk6PxcnAcnl67l/n9LC7kiMIa4vK2gUCMecuU=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
 github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0=

diff --git a/main.go b/main.go
@@ -86,6 +86,11 @@ func main() {
 			Value: false,
 			Usage: "If using an Amino DHT client should the libp2p host be shared with the data downloading host",
 		},
+		&cli.StringFlag{
+			Name:  "denylists",
+			Value: "https://denyli.st/badbits.deny",
+			Usage: "Denylist subscriptions (comma-separated)",
+		},
 	}
 
 	app.Name = "rainbow"
@@ -108,6 +113,7 @@ func main() {
 			KuboRPCURLs:     getEnvs(EnvKuboRPC, DefaultKuboRPC),
 			DHTSharedHost:   cctx.Bool("dht-fallback-shared-host"),
 			DNSCache:        cdns,
+			DenylistSubs:    strings.Split(cctx.String("denylists"), ","),
 		})
 		if err != nil {
 			return err
@@ -183,6 +189,9 @@ func main() {
 		log.Printf("Closing servers...")
 		go gatewaySrv.Close()
 		go apiSrv.Close()
+		for _, sub := range gnd.denylistSubs {
+			sub.Stop()
+		}
 		wg.Wait()
 		return nil
 	}

diff --git a/setup.go b/setup.go
@@ -6,11 +6,12 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	badger "github.com/dgraph-io/badger/v4"
 	options "github.com/dgraph-io/badger/v4/options"
-	"github.com/ipfs-shipyard/nopfs"
+	nopfs "github.com/ipfs-shipyard/nopfs"
 	nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs"
 	bsclient "github.com/ipfs/boxo/bitswap/client"
 	bsnet "github.com/ipfs/boxo/bitswap/network"
@@ -71,6 +72,9 @@ type Node struct {
 	kuboRPCs []string
 
 	bwc *metrics.BandwidthCounter
+
+	denylistSubs []*nopfs.HTTPSubscriber
+	blocker      *nopfs.Blocker
 }
 
 type Config struct {
@@ -91,6 +95,8 @@ type Config struct {
 	KuboRPCURLs   []string
 	DHTSharedHost bool
 	DNSCache      *cachedDNS
+
+	DenylistSubs []string
 }
 
 func Setup(ctx context.Context, cfg Config) (*Node, error) {
@@ -287,7 +293,18 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 	)
 	bn.Start(bswap)
 
-	files, err := nopfs.GetDenylistFiles()
+	err = os.Mkdir("denylists", 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	var denylists []*nopfs.HTTPSubscriber
+	for _, dl := range cfg.DenylistSubs {
+		s := nopfs.NewHTTPSubscriber(dl, filepath.Join("denylists", filepath.Base(dl)), time.Minute)
+		denylists = append(denylists, s)
+	}
+
+	files, err := nopfs.GetDenylistFilesInDir("denylists")
 	if err != nil {
 		return nil, err
 	}
@@ -324,16 +341,18 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 	r = nopfsipfs.WrapResolver(r, blocker)
 
 	return &Node{
-		host:       h,
-		blockstore: blkst,
-		datastore:  ds,
-		bsClient:   bswap,
-		ns:         ns,
-		vs:         vs,
-		bsrv:       bsrv,
-		resolver:   r,
-		bwc:        bwc,
-		kuboRPCs:   cfg.KuboRPCURLs,
+		host:         h,
+		blockstore:   blkst,
+		datastore:    ds,
+		bsClient:     bswap,
+		ns:           ns,
+		vs:           vs,
+		bsrv:         bsrv,
+		resolver:     r,
+		bwc:          bwc,
+		kuboRPCs:     cfg.KuboRPCURLs,
+		blocker:      blocker,
+		denylistSubs: denylists,
 	}, nil
 }