diff --git a/go.mod b/go.mod index 63b9915..18c0b06 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.20 require ( github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3 github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 2661b46..5dbe84a 100644 --- a/go.sum +++ b/go.sum @@ -287,10 +287,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 h1:IkOu07ym9Pi+O5Xl50CpMjvqULLFatzNY3+dyI54swM= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 h1:gMcb3mZGIzSngq4oGI2L/j0X0kGqvEuaNm2dpqPTtbU= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3 h1:eB1izctgB0japd/TG9hGMblUlwF82lwrTo64cwNl5Po= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016135131-28abafa61de3/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3 h1:4sJsKDnk6PxcnAcnl67l/n9LC7kiMIa4vK2gUCMecuU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016135131-28abafa61de3/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= diff --git a/main.go b/main.go index cc536f2..e82710f 100644 --- a/main.go +++ b/main.go @@ -86,6 +86,11 @@ func main() { Value: false, Usage: "If using an Amino DHT client should the libp2p host be shared with the data downloading host", }, + &cli.StringFlag{ + Name: "denylists", + Value: "https://denyli.st/badbits.deny", + Usage: "Denylist subscriptions (comma-separated)", + }, } app.Name = "rainbow" @@ -108,6 +113,7 @@ func main() { KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), DNSCache: cdns, + DenylistSubs: strings.Split(cctx.String("denylists"), ","), }) if err != nil { return err @@ -183,6 +189,9 @@ func main() { log.Printf("Closing servers...") go gatewaySrv.Close() go apiSrv.Close() + for _, sub := range gnd.denylistSubs { + sub.Stop() + } wg.Wait() return nil } diff --git a/setup.go b/setup.go index 32b09f5..1ca1071 100644 --- a/setup.go +++ b/setup.go @@ -6,11 +6,12 @@ import ( "fmt" "net/http" "os" + "path/filepath" "time" badger "github.com/dgraph-io/badger/v4" options "github.com/dgraph-io/badger/v4/options" - "github.com/ipfs-shipyard/nopfs" + nopfs "github.com/ipfs-shipyard/nopfs" nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs" bsclient "github.com/ipfs/boxo/bitswap/client" bsnet "github.com/ipfs/boxo/bitswap/network" @@ -71,6 +72,9 @@ type Node struct { kuboRPCs []string bwc *metrics.BandwidthCounter + + denylistSubs []*nopfs.HTTPSubscriber + blocker *nopfs.Blocker } type Config struct { @@ -91,6 +95,8 @@ type Config struct { KuboRPCURLs []string DHTSharedHost bool DNSCache *cachedDNS + + DenylistSubs []string } func Setup(ctx context.Context, cfg Config) (*Node, error) { @@ -287,7 +293,18 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { ) bn.Start(bswap) - files, err := nopfs.GetDenylistFiles() + err = os.Mkdir("denylists", 0755) + if err != nil { + return nil, err + } + + var denylists []*nopfs.HTTPSubscriber + for _, dl := range cfg.DenylistSubs { + s := nopfs.NewHTTPSubscriber(dl, filepath.Join("denylists", filepath.Base(dl)), time.Minute) + denylists = append(denylists, s) + } + + files, err := nopfs.GetDenylistFilesInDir("denylists") if err != nil { return nil, err } @@ -324,16 +341,18 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { r = nopfsipfs.WrapResolver(r, blocker) return &Node{ - host: h, - blockstore: blkst, - datastore: ds, - bsClient: bswap, - ns: ns, - vs: vs, - bsrv: bsrv, - resolver: r, - bwc: bwc, - kuboRPCs: cfg.KuboRPCURLs, + host: h, + blockstore: blkst, + datastore: ds, + bsClient: bswap, + ns: ns, + vs: vs, + bsrv: bsrv, + resolver: r, + bwc: bwc, + kuboRPCs: cfg.KuboRPCURLs, + blocker: blocker, + denylistSubs: denylists, }, nil }