From 3c69a94e9b4d78f3784a2d35a39b7e24cd80cb0a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 11 Oct 2023 19:54:20 +0200 Subject: [PATCH 1/6] Add integrated nopfs-content blocking to the gateway This is pending some go.mod bubbling. --- go.mod | 11 +++++++---- go.sum | 12 +++++++++--- handlers.go | 15 ++++++++++++--- setup.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 395e260..6dfaff9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.20 require ( github.com/dgraph-io/badger/v4 v4.2.0 - github.com/ipfs/boxo v0.13.1 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 + github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 @@ -12,6 +14,9 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 + github.com/ipfs/go-unixfsnode v1.7.1 + github.com/ipld/go-codec-dagpb v1.6.0 + github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.30.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-record v0.2.0 @@ -52,6 +57,7 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -85,10 +91,7 @@ require ( github.com/ipfs/go-ipld-format v0.5.0 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-unixfsnode v1.7.1 // indirect github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect - github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect diff --git a/go.sum b/go.sum index 3e4eac2..9d9f48d 100644 --- a/go.sum +++ b/go.sum @@ -147,7 +147,8 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -286,10 +287,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 h1:IkOu07ym9Pi+O5Xl50CpMjvqULLFatzNY3+dyI54swM= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 h1:gMcb3mZGIzSngq4oGI2L/j0X0kGqvEuaNm2dpqPTtbU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.1 h1:nQ5oQzcMZR3oL41REJDcTbrvDvuZh3J9ckc9+ILeRQI= -github.com/ipfs/boxo v0.13.1/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3 h1:sgrhALL6mBoZsNvJ2zUcITcN6IW3y14ej6w7gv5RcOI= +github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -914,6 +919,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/handlers.go b/handlers.go index 49ccde8..f9ccee1 100644 --- a/handlers.go +++ b/handlers.go @@ -12,8 +12,8 @@ import ( _ "net/http/pprof" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/path" servertiming "github.com/mitchellh/go-server-timing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -174,7 +174,12 @@ func newKuboRPCHandler(endpoints []string) http.Handler { // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303 redirectToGateway := func(format string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - path := path.New(r.URL.Query().Get("arg")) + path, err := path.NewPath(r.URL.Query().Get("arg")) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + return + } url := path.String() if format != "" { url += "?format=" + format @@ -189,7 +194,11 @@ func newKuboRPCHandler(endpoints []string) http.Handler { mux.HandleFunc("/api/v0/dag/export", redirectToGateway("car")) mux.HandleFunc("/api/v0/block/get", redirectToGateway("raw")) mux.HandleFunc("/api/v0/dag/get", func(w http.ResponseWriter, r *http.Request) { - path := path.New(r.URL.Query().Get("arg")) + path, err := path.NewPath(r.URL.Query().Get("arg")) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + } codec := r.URL.Query().Get("output-codec") if codec == "" { codec = "dag-json" diff --git a/setup.go b/setup.go index 2e924d4..18e313a 100644 --- a/setup.go +++ b/setup.go @@ -10,13 +10,17 @@ import ( badger "github.com/dgraph-io/badger/v4" options "github.com/dgraph-io/badger/v4/options" + "github.com/ipfs-shipyard/nopfs" + nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs" bsclient "github.com/ipfs/boxo/bitswap/client" bsnet "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" + "github.com/ipfs/boxo/path/resolver" routingv1client "github.com/ipfs/boxo/routing/http/client" httpcontentrouter "github.com/ipfs/boxo/routing/http/contentrouter" "github.com/ipfs/go-cid" @@ -25,6 +29,11 @@ import ( levelds "github.com/ipfs/go-ds-leveldb" metri "github.com/ipfs/go-metrics-interface" mprome "github.com/ipfs/go-metrics-prometheus" + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/fullrt" @@ -57,6 +66,7 @@ type Node struct { blockstore blockstore.Blockstore bsClient *bsclient.Client bsrv blockservice.BlockService + resolver resolver.Resolver ns namesys.NameSystem kuboRPCs []string @@ -260,6 +270,15 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { bswap := bsclient.New(bsctx, bn, blkst) bn.Start(bswap) + files, err := nopfs.GetDenylistFiles() + if err != nil { + return nil, err + } + blocker, err := nopfs.NewBlocker(files) + if err != nil { + return nil, err + } + bsrv := blockservice.New(blkst, bswap, // if we are doing things right, our bitswap wantlists should // not have blocks that we already have (see @@ -269,6 +288,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { // before writing new blocks. blockservice.WriteThrough(), ) + bsrv = nopfsipfs.WrapBlockService(bsrv, blocker) dns, err := gateway.NewDNSResolver(nil) if err != nil { @@ -278,6 +298,18 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { if err != nil { return nil, err } + ns = nopfsipfs.WrapNameSystem(ns, blocker) + + fetcherConfig := bsfetcher.NewFetcherConfig(bsrv) + fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + r := resolver.NewBasicResolver(fetcher) + r = nopfsipfs.WrapResolver(r, blocker) return &Node{ host: h, @@ -287,6 +319,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { ns: ns, vs: vs, bsrv: bsrv, + resolver: r, bwc: bwc, kuboRPCs: cfg.KuboRPCURLs, }, nil From 70d52021e9133266cf11ec0b8e2da629dd6079fb Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 12 Oct 2023 19:23:53 +0200 Subject: [PATCH 2/6] No need to set the Reifier in the fetcher as it is the default already --- setup.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/setup.go b/setup.go index 18e313a..866cfbc 100644 --- a/setup.go +++ b/setup.go @@ -29,11 +29,6 @@ import ( levelds "github.com/ipfs/go-ds-leveldb" metri "github.com/ipfs/go-metrics-interface" mprome "github.com/ipfs/go-metrics-prometheus" - "github.com/ipfs/go-unixfsnode" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/fullrt" @@ -300,14 +295,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } ns = nopfsipfs.WrapNameSystem(ns, blocker) - fetcherConfig := bsfetcher.NewFetcherConfig(bsrv) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + fetcher := bsfetcher.NewFetcherConfig(bsrv) r := resolver.NewBasicResolver(fetcher) r = nopfsipfs.WrapResolver(r, blocker) From ca1473387c53ada2fdacbd9de6ff9fbf2c35c1c8 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 12 Oct 2023 19:27:01 +0200 Subject: [PATCH 3/6] Missing return on path-parsing errors --- handlers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/handlers.go b/handlers.go index f9ccee1..7e7cb7e 100644 --- a/handlers.go +++ b/handlers.go @@ -198,6 +198,7 @@ func newKuboRPCHandler(endpoints []string) http.Handler { if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) + return } codec := r.URL.Query().Get("output-codec") if codec == "" { From 8f160d3046fd342cb6da8f2917a2715173e738e4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 13 Oct 2023 16:03:26 +0200 Subject: [PATCH 4/6] fix fetcher initialization --- setup.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.go b/setup.go index 866cfbc..bf7a319 100644 --- a/setup.go +++ b/setup.go @@ -29,6 +29,8 @@ import ( levelds "github.com/ipfs/go-ds-leveldb" metri "github.com/ipfs/go-metrics-interface" mprome "github.com/ipfs/go-metrics-prometheus" + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" "github.com/libp2p/go-libp2p" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/fullrt" @@ -295,7 +297,9 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } ns = nopfsipfs.WrapNameSystem(ns, blocker) - fetcher := bsfetcher.NewFetcherConfig(bsrv) + fetcherCfg := bsfetcher.NewFetcherConfig(bsrv) + fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) + fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) r := resolver.NewBasicResolver(fetcher) r = nopfsipfs.WrapResolver(r, blocker) From 46ef95fecdd0e5595196f26ed5acfb51a3811ad8 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 13 Oct 2023 16:37:05 +0200 Subject: [PATCH 5/6] prefix paths with /ipfs/ prefix if they don't have any prefix --- handlers.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/handlers.go b/handlers.go index 7e7cb7e..8a4c59c 100644 --- a/handlers.go +++ b/handlers.go @@ -165,6 +165,17 @@ func setupGatewayHandler(nd *Node) (http.Handler, error) { return handler, nil } +// Prefixes with /ipfs/ unprefixed paths. +func prefixPath(p string) string { + if len(p) == 0 { + return p + } + if p[0] != '/' { + return "/ipfs/" + p + } + return p // let NewPath handle whatever issues from here. +} + func newKuboRPCHandler(endpoints []string) http.Handler { mux := http.NewServeMux() @@ -174,7 +185,7 @@ func newKuboRPCHandler(endpoints []string) http.Handler { // - https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303 redirectToGateway := func(format string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - path, err := path.NewPath(r.URL.Query().Get("arg")) + path, err := path.NewPath(prefixPath(r.URL.Query().Get("arg"))) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -194,7 +205,7 @@ func newKuboRPCHandler(endpoints []string) http.Handler { mux.HandleFunc("/api/v0/dag/export", redirectToGateway("car")) mux.HandleFunc("/api/v0/block/get", redirectToGateway("raw")) mux.HandleFunc("/api/v0/dag/get", func(w http.ResponseWriter, r *http.Request) { - path, err := path.NewPath(r.URL.Query().Get("arg")) + path, err := path.NewPath(prefixPath(r.URL.Query().Get("arg"))) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) From b6c9163bf87901563e52be6c3b017908dbd829cc Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 13 Oct 2023 17:23:26 +0200 Subject: [PATCH 6/6] go.mod cleanup --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6dfaff9..6f741cb 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-unixfsnode v1.7.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.30.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-record v0.2.0 @@ -92,6 +91,7 @@ require ( github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect