Merge pull request #22 from ipfs/nopfs

Add integrated nopfs-content blocking to the gateway

go.mod

@@ -4,14 +4,18 @@ go 1.20
 
 require (
 	github.com/dgraph-io/badger/v4 v4.2.0
-	github.com/ipfs/boxo v0.13.1
+	github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941
+	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71
+	github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
+	github.com/ipfs/go-unixfsnode v1.7.1
+	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/libp2p/go-libp2p v0.30.0
 	github.com/libp2p/go-libp2p-kad-dht v0.23.0
 	github.com/libp2p/go-libp2p-record v0.2.0
@@ -52,6 +56,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -85,9 +90,7 @@ require (
 	github.com/ipfs/go-ipld-format v0.5.0 // indirect
 	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-unixfsnode v1.7.1 // indirect
 	github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect
-	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect

go.sum

@@ -147,7 +147,8 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
 github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -286,10 +287,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
 github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941 h1:IkOu07ym9Pi+O5Xl50CpMjvqULLFatzNY3+dyI54swM=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231012170252-ab19c8a36941/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71 h1:gMcb3mZGIzSngq4oGI2L/j0X0kGqvEuaNm2dpqPTtbU=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231012170819-d73899390a71/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.13.1 h1:nQ5oQzcMZR3oL41REJDcTbrvDvuZh3J9ckc9+ILeRQI=
-github.com/ipfs/boxo v0.13.1/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
+github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3 h1:sgrhALL6mBoZsNvJ2zUcITcN6IW3y14ej6w7gv5RcOI=
+github.com/ipfs/boxo v0.13.2-0.20231012132507-6602207a8fa3/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -914,6 +919,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

handlers.go

@@ -12,8 +12,8 @@ import (
 
 	_ "net/http/pprof"
 
-	"github.com/ipfs/boxo/coreiface/path"
 	"github.com/ipfs/boxo/gateway"
+	"github.com/ipfs/boxo/path"
 	servertiming "github.com/mitchellh/go-server-timing"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -165,6 +165,17 @@ func setupGatewayHandler(nd *Node) (http.Handler, error) {
 	return handler, nil
 }
 
+// Prefixes with /ipfs/ unprefixed paths.
+func prefixPath(p string) string {
+	if len(p) == 0 {
+		return p
+	}
+	if p[0] != '/' {
+		return "/ipfs/" + p
+	}
+	return p // let NewPath handle whatever issues from here.
+}
+
 func newKuboRPCHandler(endpoints []string) http.Handler {
 	mux := http.NewServeMux()
 
@@ -174,7 +185,12 @@ func newKuboRPCHandler(endpoints []string) http.Handler {
 	// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303
 	redirectToGateway := func(format string) http.HandlerFunc {
 		return func(w http.ResponseWriter, r *http.Request) {
-			path := path.New(r.URL.Query().Get("arg"))
+			path, err := path.NewPath(prefixPath(r.URL.Query().Get("arg")))
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				w.Write([]byte(err.Error()))
+				return
+			}
 			url := path.String()
 			if format != "" {
 				url += "?format=" + format
@@ -189,7 +205,12 @@ func newKuboRPCHandler(endpoints []string) http.Handler {
 	mux.HandleFunc("/api/v0/dag/export", redirectToGateway("car"))
 	mux.HandleFunc("/api/v0/block/get", redirectToGateway("raw"))
 	mux.HandleFunc("/api/v0/dag/get", func(w http.ResponseWriter, r *http.Request) {
-		path := path.New(r.URL.Query().Get("arg"))
+		path, err := path.NewPath(prefixPath(r.URL.Query().Get("arg")))
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			w.Write([]byte(err.Error()))
+			return
+		}
 		codec := r.URL.Query().Get("output-codec")
 		if codec == "" {
 			codec = "dag-json"

setup.go

@@ -10,13 +10,17 @@ import (
 
 	badger "github.com/dgraph-io/badger/v4"
 	options "github.com/dgraph-io/badger/v4/options"
+	"github.com/ipfs-shipyard/nopfs"
+	nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs"
 	bsclient "github.com/ipfs/boxo/bitswap/client"
 	bsnet "github.com/ipfs/boxo/bitswap/network"
 	"github.com/ipfs/boxo/blockservice"
 	"github.com/ipfs/boxo/blockstore"
+	bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
 	"github.com/ipfs/boxo/gateway"
 	"github.com/ipfs/boxo/ipns"
 	"github.com/ipfs/boxo/namesys"
+	"github.com/ipfs/boxo/path/resolver"
 	routingv1client "github.com/ipfs/boxo/routing/http/client"
 	httpcontentrouter "github.com/ipfs/boxo/routing/http/contentrouter"
 	"github.com/ipfs/go-cid"
@@ -25,6 +29,8 @@ import (
 	levelds "github.com/ipfs/go-ds-leveldb"
 	metri "github.com/ipfs/go-metrics-interface"
 	mprome "github.com/ipfs/go-metrics-prometheus"
+	"github.com/ipfs/go-unixfsnode"
+	dagpb "github.com/ipld/go-codec-dagpb"
 	"github.com/libp2p/go-libp2p"
 	dht "github.com/libp2p/go-libp2p-kad-dht"
 	"github.com/libp2p/go-libp2p-kad-dht/fullrt"
@@ -57,6 +63,7 @@ type Node struct {
 	blockstore blockstore.Blockstore
 	bsClient   *bsclient.Client
 	bsrv       blockservice.BlockService
+	resolver   resolver.Resolver
 
 	ns       namesys.NameSystem
 	kuboRPCs []string
@@ -260,6 +267,15 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 	bswap := bsclient.New(bsctx, bn, blkst)
 	bn.Start(bswap)
 
+	files, err := nopfs.GetDenylistFiles()
+	if err != nil {
+		return nil, err
+	}
+	blocker, err := nopfs.NewBlocker(files)
+	if err != nil {
+		return nil, err
+	}
+
 	bsrv := blockservice.New(blkst, bswap,
 		// if we are doing things right, our bitswap wantlists should
 		// not have blocks that we already have (see
@@ -269,6 +285,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 		// before writing new blocks.
 		blockservice.WriteThrough(),
 	)
+	bsrv = nopfsipfs.WrapBlockService(bsrv, blocker)
 
 	dns, err := gateway.NewDNSResolver(nil)
 	if err != nil {
@@ -278,6 +295,13 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 	if err != nil {
 		return nil, err
 	}
+	ns = nopfsipfs.WrapNameSystem(ns, blocker)
+
+	fetcherCfg := bsfetcher.NewFetcherConfig(bsrv)
+	fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
+	fetcher := fetcherCfg.WithReifier(unixfsnode.Reify)
+	r := resolver.NewBasicResolver(fetcher)
+	r = nopfsipfs.WrapResolver(r, blocker)
 
 	return &Node{
 		host:       h,
@@ -287,6 +311,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) {
 		ns:         ns,
 		vs:         vs,
 		bsrv:       bsrv,
+		resolver:   r,
 		bwc:        bwc,
 		kuboRPCs:   cfg.KuboRPCURLs,
 	}, nil