From 86278bdb55ecccbd16db8818e5f8d3f8a3ad9a76 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 12 May 2023 10:28:36 +0200 Subject: [PATCH] feat(gateway): preview dag-cbor/-json with links --- gateway/assets/assets.go | 5 +- gateway/assets/dag.html | 27 +++++ gateway/assets/node.go | 110 ++++++++++++++++++ gateway/assets/style.css | 36 +++++- gateway/assets/templates.go | 6 + ...wmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block | Bin 0 -> 742 bytes ...k2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block | Bin 0 -> 238 bytes ...nemm6idx4r2n6u3apmtcrxlqwuapgjsciihy.block | Bin 0 -> 332 bytes gateway/assets/test/main.go | 89 ++++++++++++-- gateway/handler.go | 21 ++++ gateway/handler_codec.go | 41 +++++-- gateway/handler_unixfs_dir.go | 26 +---- 12 files changed, 313 insertions(+), 48 deletions(-) create mode 100644 gateway/assets/node.go create mode 100644 gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block create mode 100644 gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block create mode 100644 gateway/assets/test/dag/bafyreihnpl7ami7esahkfdnemm6idx4r2n6u3apmtcrxlqwuapgjsciihy.block diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index dc4748b8c..06a88e09e 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -82,6 +82,8 @@ func initTemplates() { type GlobalData struct { SupportURL string + GatewayURL string + DNSLink bool } type DagTemplateData struct { @@ -90,6 +92,7 @@ type DagTemplateData struct { CID string CodecName string CodecHex string + Node *ParsedNode } type ErrorTemplateData struct { @@ -101,8 +104,6 @@ type ErrorTemplateData struct { type DirectoryTemplateData struct { GlobalData - GatewayURL string - DNSLink bool Listing []DirectoryItem Size string Path string diff --git a/gateway/assets/dag.html b/gateway/assets/dag.html index be739aaee..3a4c0dfc8 100644 --- a/gateway/assets/dag.html +++ b/gateway/assets/dag.html @@ -28,6 +28,33 @@
  • Valid DAG-CBOR (specs at IPLD and IANA)
    • + {{ with .Node }} +
    +
    + DAG Preview +
    + {{ template "node" (args $ .) }} +
    + {{ end }} + +{{ define "node" }} + {{ $root := index . 0 }} + {{ $node := index . 1 }} + {{ if len $node.Values }} +
    + {{ range $index, $key := $node.Keys }} + {{ template "node" (args $root $key) }} + {{ template "node" (args $root (index $node.Values $index)) }} + {{ end }} +
    + {{ else }} +
    + {{ with $node.CID }}{{ end }} + {{- $node.Value -}} + {{ with $node.CID }}{{ end }} +
    + {{ end }} +{{ end }} diff --git a/gateway/assets/node.go b/gateway/assets/node.go new file mode 100644 index 000000000..692597910 --- /dev/null +++ b/gateway/assets/node.go @@ -0,0 +1,110 @@ +package assets + +import ( + "fmt" + "html/template" + + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +type ParsedNode struct { + Keys []*ParsedNode + Values []*ParsedNode + Value string + CID string +} + +func ParseNode(node datamodel.Node) (*ParsedNode, error) { + dag := &ParsedNode{} + + switch node.Kind() { + case datamodel.Kind_Map: + it := node.MapIterator() + + for !it.Done() { + k, v, err := it.Next() + if err != nil { + return nil, err + } + + kd, err := ParseNode(k) + if err != nil { + return nil, err + } + + vd, err := ParseNode(v) + if err != nil { + return nil, err + } + + dag.Keys = append(dag.Keys, kd) + dag.Values = append(dag.Values, vd) + } + case datamodel.Kind_List: + it := node.ListIterator() + for !it.Done() { + k, v, err := it.Next() + if err != nil { + return nil, err + } + + vd, err := ParseNode(v) + if err != nil { + return nil, err + } + + dag.Keys = append(dag.Keys, &ParsedNode{Value: fmt.Sprint(k)}) + dag.Values = append(dag.Values, vd) + } + case datamodel.Kind_Bool: + v, err := node.AsBool() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%t", v) + case datamodel.Kind_Int: + v, err := node.AsInt() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%d", v) + case datamodel.Kind_Float: + v, err := node.AsFloat() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%f", v) + case datamodel.Kind_String: + v, err := node.AsString() + if err != nil { + return nil, err + } + dag.Value = template.HTMLEscapeString(v) + case datamodel.Kind_Bytes: + v, err := node.AsBytes() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprint(v) + case datamodel.Kind_Link: + lnk, err := node.AsLink() + if err != nil { + return nil, err + } + dag.Value = lnk.String() + + cl, isCid := lnk.(cidlink.Link) + if isCid { + dag.CID = cl.Cid.String() + } + case datamodel.Kind_Invalid: + dag.Value = "INVALID" + case datamodel.Kind_Null: + dag.Value = "NULL" + default: + dag.Value = "UNKNOWN" + } + + return dag, nil +} diff --git a/gateway/assets/style.css b/gateway/assets/style.css index d6990dfbb..cfce5de54 100644 --- a/gateway/assets/style.css +++ b/gateway/assets/style.css @@ -144,13 +144,17 @@ main section:not(:last-child) { border-bottom: 1px solid var(--dark-white); } +main section header { + background-color: var(--near-white); +} + .grid { display: grid; } .grid > div { padding: .7em; - border-top: 1px solid var(--dark-white); + border-bottom: 1px solid var(--dark-white); } .grid.dir { @@ -165,8 +169,8 @@ main section:not(:last-child) { padding-right: 1em; } -.grid.dir > div:nth-child(-n+4) { - border-top: 0; +.grid.dir > div:nth-last-child(-n+4) { + border-bottom: 0; } .grid.dir > div:nth-of-type(8n+5), @@ -176,6 +180,31 @@ main section:not(:last-child) { background-color: var(--near-white); } +.grid.dag { + grid-template-columns: max-content 1fr; +} + +.grid.dag .grid { + padding: 0; +} + +.grid.dag > div:nth-last-child(-n+2) { + border-bottom: 0; +} + +.grid.dag > div { + background: white +} + +.grid.dag > div:nth-child(4n), +.grid.dag > div:nth-child(4n+3) { + background-color: var(--near-white); +} + +section > .grid.dag > div:nth-of-type(2n+1) { + padding-left: 1em; +} + .type-icon, .type-icon > * { width: 1.15em @@ -222,4 +251,3 @@ main section:not(:last-child) { display: none; } } - diff --git a/gateway/assets/templates.go b/gateway/assets/templates.go index 73b4696ae..a7c98c221 100644 --- a/gateway/assets/templates.go +++ b/gateway/assets/templates.go @@ -24,9 +24,15 @@ func iconFromExt(filename string) string { return "ipfs-_blank" // Default is blank icon. } +// args is a helper function to allow sending more than one object to a template. +func args(args ...interface{}) []interface{} { + return args +} + var funcMap = template.FuncMap{ "iconFromExt": iconFromExt, "urlEscape": urlEscape, + "args": args, } func readFile(fs fs.FS, filename string) ([]byte, error) { diff --git a/gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block b/gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block new file mode 100644 index 0000000000000000000000000000000000000000..db8145d6d37ee56733e82ffecf44b457ca72587b GIT binary patch literal 742 zcmZo>VVHD7D?*ilu~0}skN0!<><7{Oit>UDK~j{Vw$5NxzZH z4!FWy_pi<7Y}Q{LA$NPh`aczWoBsE*&O5|&;?GArwl>>;tZ;?fjjG+x9&xsN^wUT^ zCuPmP>kGsB_Rd%H+qvWWuH2xHU2uikm&1)CU){3V^v3YtNn7T)cO^UiOzD-EuzWtF zbm5G@&*2K|GYiYk%rm@J<#=T2^Q6nC7ryWJl36rw(y5F4kHo&ty#rTh7H&+K>zG*@w#s>-*A>+ zb~1l+dwKqfMR0|x?P7Hc=hfcg+0M1zIjGV4V#hxDWzThuV*)CQ6XP{5!4*dO9ThX% z_pB|K(U<+a@7o^rh=31F1^S;YHgLX>@wogNuF%WAH~RRr1e3R>2@Pe(CRnX4Kjow5 z6#7%&<3Mc3%T__S!Xv6dt>zk>8Y>pwws(A1&;Loc({egXM$dI-gEzt5QOR(HUVZiT zIV_7WEL$Cx9?iyk^PQp3GeNN?q0F5o?M!(-C2)mKn%+VhT(XZlc^lrE9G8t`Q`D7` zo#3%^>-o-;y_cgO!WDiF;_RE-J7qybZ(`MsFny82X=}WK%JqJ!^zQPvpY6{FS12FD jbo2DJN8HD5in!KhD7JO6=o!0t<(s^_w&h8laAqL@3K2pk literal 0 HcmV?d00001 diff --git a/gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block b/gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block new file mode 100644 index 0000000000000000000000000000000000000000..5605a83bd2d5ae4a3e5225f943989b1a1a588a8f GIT binary patch literal 238 zcmZ3SlAKsloRODbq5xu~C1<3j7N@q{(27uHUh9V+ZGj1! zmMTAFIAyfThh6)Xdn?ab-jCg4&)4|F752_+T_BtJyL#@iSMB?nzIU}>-w`I%zH@$7 z&8BbX9)Fw#SNOJ(u{t4E&hGt5xy`fxug^)=d0GAEnM5sn!Jv&8F9(K zmfySc@!YefwGn$~eA#hUa!&8>N7p&>?XUfVE1XzeaXca?|LN_brJa8Y!mls{=s)rg zUa;HY$NT=xCV>fXg#tNM#pTTA+qKirPAifx;_I9{H~7M(dFOIo^|FccItc;*v6Ys; literal 0 HcmV?d00001 diff --git a/gateway/assets/test/main.go b/gateway/assets/test/main.go index d5e996055..a98fd0569 100644 --- a/gateway/assets/test/main.go +++ b/gateway/assets/test/main.go @@ -1,14 +1,30 @@ package main import ( + "embed" "fmt" "net/http" "os" "strconv" + "strings" "github.com/ipfs/boxo/gateway/assets" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/node/basicnode" + mc "github.com/multiformats/go-multicodec" + + // Ensure basic codecs are registered. + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" + _ "github.com/ipld/go-ipld-prime/codec/raw" ) +//go:embed dag/*.block +var embeds embed.FS + const ( testPath = "/ipfs/QmFooBarQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7/a/b/c" ) @@ -16,9 +32,9 @@ const ( var directoryTestData = assets.DirectoryTemplateData{ GlobalData: assets.GlobalData{ SupportURL: "http://example.com", + GatewayURL: "//localhost:3000", + DNSLink: true, }, - GatewayURL: "//localhost:3000", - DNSLink: true, Listing: []assets.DirectoryItem{{ Size: "25 MiB", Name: "short-film.mov", @@ -59,14 +75,58 @@ var directoryTestData = assets.DirectoryTemplateData{ Hash: "QmFooBazBar2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7", } -var dagTestData = assets.DagTemplateData{ - GlobalData: assets.GlobalData{ - SupportURL: "http://example.com", - }, - Path: "/ipfs/baguqeerabn4wonmz6icnk7dfckuizcsf4e4igua2ohdboecku225xxmujepa", - CID: "baguqeerabn4wonmz6icnk7dfckuizcsf4e4igua2ohdboecku225xxmujepa", - CodecName: "dag-json", - CodecHex: "0x129", +var dagTestData = map[string]*assets.DagTemplateData{} + +func loadDagTestData() { + entries, err := embeds.ReadDir("dag") + if err != nil { + panic(err) + } + + for _, entry := range entries { + cidStr := strings.TrimSuffix(entry.Name(), ".block") + cid, err := cid.Decode(cidStr) + if err != nil { + panic(err) + } + + f, err := embeds.Open("dag/" + entry.Name()) + if err != nil { + panic(err) + } + + codec := cid.Prefix().Codec + decoder, err := multicodec.LookupDecoder(codec) + if err != nil { + panic(err) + } + + node := basicnode.Prototype.Any.NewBuilder() + err = decoder(node, f) + if err != nil { + panic(err) + } + + cidCodec := mc.Code(cid.Prefix().Codec) + + dag, err := assets.ParseNode(node.Build()) + if err != nil { + panic(err) + } + + dagTestData[cid.String()] = &assets.DagTemplateData{ + GlobalData: assets.GlobalData{ + SupportURL: "http://example.com", + GatewayURL: "//localhost:3000", + DNSLink: true, + }, + Path: "/ipfs/" + cid.String(), + CID: cid.String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: dag, + } + } } func init() { @@ -80,6 +140,8 @@ func init() { ShortHash: "QmbW\u2026sMnR", }) } + + loadDagTestData() } func runTemplate(w http.ResponseWriter, filename string, data interface{}) { @@ -101,7 +163,12 @@ func main() { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/dag": - runTemplate(w, "dag.html", dagTestData) + cid := r.URL.Query().Get("cid") + if cid == "" { + cid = "bafyreihnpl7ami7esahkfdnemm6idx4r2n6u3apmtcrxlqwuapgjsciihy" + } + + runTemplate(w, "dag.html", dagTestData[cid]) case "/directory": runTemplate(w, "directory.html", directoryTestData) case "/error": diff --git a/gateway/handler.go b/gateway/handler.go index af4fc78b1..b79d0c914 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -17,6 +17,7 @@ import ( "time" ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/gateway/assets" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p/core/peer" @@ -832,3 +833,23 @@ func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Requ return true } + +// getTemplateGlobalData returns the global data necessary by most templates. +func (i *handler) getTemplateGlobalData(r *http.Request, contentPath ipath.Path) assets.GlobalData { + // gatewayURL is used to link to other root CIDs. THis will be blank unless + // subdomain or DNSLink resolution is being used for this request. + var gatewayURL string + if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok { + gatewayURL = "//" + h + } else { + gatewayURL = "" + } + + dnsLink := assets.HasDNSLinkOrigin(gatewayURL, contentPath.String()) + + return assets.GlobalData{ + SupportURL: i.config.SupportURL, + GatewayURL: gatewayURL, + DNSLink: dnsLink, + } +} diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index f2d9691d3..2d9f58ab6 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -120,7 +120,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt download := r.URL.Query().Get("download") == "true" if isDAG && acceptsHTML && !download { - return i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) + return i.serveCodecHTML(ctx, w, r, blockCid, blockData, resolvedPath, contentPath) } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. @@ -153,7 +153,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return i.serveCodecConverted(ctx, w, r, blockCid, blockData, contentPath, toCodec, modtime, begin) } -func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { +func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { // A HTML directory index will be presented, be sure to set the correct // type instead of relying on autodetection (which may fail). w.Header().Set("Content-Type", "text/html") @@ -172,13 +172,12 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) if err := assets.DagTemplate.Execute(w, assets.DagTemplateData{ - GlobalData: assets.GlobalData{ - SupportURL: i.config.SupportURL, - }, - Path: contentPath.String(), - CID: resolvedPath.Cid().String(), - CodecName: cidCodec.String(), - CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + GlobalData: i.getTemplateGlobalData(r, contentPath), + Path: contentPath.String(), + CID: resolvedPath.Cid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: parseNode(blockCid, blockData), }); err != nil { err = fmt.Errorf("failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw: %w", err) i.webError(w, r, err, http.StatusInternalServerError) @@ -188,6 +187,30 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * return true } +// parseNode does a best effort attempt to parse this request's block such that +// a preview can be displayed in the gateway. If something fails along the way, +// returns nil, therefore not displaying the preview. +func parseNode(blockCid cid.Cid, blockData io.ReadSeekCloser) *assets.ParsedNode { + codec := blockCid.Prefix().Codec + decoder, err := multicodec.LookupDecoder(codec) + if err != nil { + return nil + } + + nodeBuilder := basicnode.Prototype.Any.NewBuilder() + err = decoder(nodeBuilder, blockData) + if err != nil { + return nil + } + + parsedNode, err := assets.ParseNode(nodeBuilder.Build()) + if err != nil { + return nil + } + + return parsedNode +} + // serveCodecRaw returns the raw block without any conversion func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockData io.ReadSeekCloser, contentPath ipath.Path, name string, modtime, begin time.Time) bool { // ServeContent will take care of diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index a227e231d..82c2a297e 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -178,39 +178,21 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } size := humanize.Bytes(directoryMetadata.dagSize) - hash := resolvedPath.Cid().String() - - // Gateway root URL to be used when linking to other rootIDs. - // This will be blank unless subdomain or DNSLink resolution is being used - // for this request. - var gwURL string - - // Get gateway hostname and build gateway URL. - if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok { - gwURL = "//" + h - } else { - gwURL = "" - } - - dnslink := assets.HasDNSLinkOrigin(gwURL, contentPath.String()) + globalData := i.getTemplateGlobalData(r, contentPath) // See comment above where originalUrlPath is declared. tplData := assets.DirectoryTemplateData{ - GlobalData: assets.GlobalData{ - SupportURL: i.config.SupportURL, - }, - GatewayURL: gwURL, - DNSLink: dnslink, + GlobalData: globalData, Listing: dirListing, Size: size, Path: contentPath.String(), - Breadcrumbs: assets.Breadcrumbs(contentPath.String(), dnslink), + Breadcrumbs: assets.Breadcrumbs(contentPath.String(), globalData.DNSLink), BackLink: backLink, Hash: hash, } - logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) + logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) if err := assets.DirectoryTemplate.Execute(w, tplData); err != nil { i.webError(w, r, err, http.StatusInternalServerError)