diff --git a/CHANGELOG.md b/CHANGELOG.md index 73348f885..efc2c6c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The following emojis are used to highlight certain changes: - New human-friendly error messages. - Updated, higher-definition icons in directory listings. - Customizable menu items next to "About IPFS" and "Install IPFS". + - Valid DAG-CBOR and DAG-JSON blocks now provide a preview, where links can be followed. ## [0.8.0] - 2023-04-05 ### Added diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index 5283ef502..f411a47e0 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -86,7 +86,9 @@ type MenuItem struct { } type GlobalData struct { - Menu []MenuItem + Menu []MenuItem + GatewayURL string + DNSLink bool } type DagTemplateData struct { @@ -95,6 +97,7 @@ type DagTemplateData struct { CID string CodecName string CodecHex string + Node *ParsedNode } type ErrorTemplateData struct { @@ -106,8 +109,6 @@ type ErrorTemplateData struct { type DirectoryTemplateData struct { GlobalData - GatewayURL string - DNSLink bool Listing []DirectoryItem Size string Path string diff --git a/gateway/assets/dag.html b/gateway/assets/dag.html index aa2359e0d..b76a97054 100644 --- a/gateway/assets/dag.html +++ b/gateway/assets/dag.html @@ -13,10 +13,10 @@
- CID: {{.CID}} + CID: {{ .CID }}
- Codec: {{.CodecName}} ({{.CodecHex}}) + Codec: {{ .CodecName }} ({{ .CodecHex }})
@@ -28,6 +28,33 @@
  • Valid DAG-CBOR (specs at IPLD and IANA)
    • + {{ with .Node }} +
    +
    + {{ $.CodecName }} preview +
    + {{ template "node" (args $ .) }} +
    + {{ end }}
    + +{{ define "node" }} + {{ $root := index . 0 }} + {{ $node := index . 1 }} + {{ if len $node.Values }} +
    + {{ range $index, $key := $node.Keys }} + {{ template "node" (args $root $key) }} + {{ template "node" (args $root (index $node.Values $index)) }} + {{ end }} +
    + {{ else }} +
    + {{ with $node.CID }}{{ end }} + {{- $node.Value -}} + {{ with $node.CID }}{{ end }} +
    + {{ end }} +{{ end }} diff --git a/gateway/assets/node.go b/gateway/assets/node.go new file mode 100644 index 000000000..b90441502 --- /dev/null +++ b/gateway/assets/node.go @@ -0,0 +1,110 @@ +package assets + +import ( + "encoding/hex" + "fmt" + + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +type ParsedNode struct { + Keys []*ParsedNode + Values []*ParsedNode + Value string + CID string +} + +func ParseNode(node datamodel.Node) (*ParsedNode, error) { + dag := &ParsedNode{} + + switch node.Kind() { + case datamodel.Kind_Map: + it := node.MapIterator() + + for !it.Done() { + k, v, err := it.Next() + if err != nil { + return nil, err + } + + kd, err := ParseNode(k) + if err != nil { + return nil, err + } + + vd, err := ParseNode(v) + if err != nil { + return nil, err + } + + dag.Keys = append(dag.Keys, kd) + dag.Values = append(dag.Values, vd) + } + case datamodel.Kind_List: + it := node.ListIterator() + for !it.Done() { + k, v, err := it.Next() + if err != nil { + return nil, err + } + + vd, err := ParseNode(v) + if err != nil { + return nil, err + } + + dag.Keys = append(dag.Keys, &ParsedNode{Value: fmt.Sprintf("%d", k)}) + dag.Values = append(dag.Values, vd) + } + case datamodel.Kind_Bool: + v, err := node.AsBool() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%t", v) + case datamodel.Kind_Int: + v, err := node.AsInt() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%d", v) + case datamodel.Kind_Float: + v, err := node.AsFloat() + if err != nil { + return nil, err + } + dag.Value = fmt.Sprintf("%f", v) + case datamodel.Kind_String: + v, err := node.AsString() + if err != nil { + return nil, err + } + dag.Value = v + case datamodel.Kind_Bytes: + v, err := node.AsBytes() + if err != nil { + return nil, err + } + dag.Value = hex.EncodeToString(v) + case datamodel.Kind_Link: + lnk, err := node.AsLink() + if err != nil { + return nil, err + } + dag.Value = lnk.String() + + cl, isCid := lnk.(cidlink.Link) + if isCid { + dag.CID = cl.Cid.String() + } + case datamodel.Kind_Invalid: + dag.Value = "INVALID" + case datamodel.Kind_Null: + dag.Value = "NULL" + default: + dag.Value = "UNKNOWN" + } + + return dag, nil +} diff --git a/gateway/assets/style.css b/gateway/assets/style.css index d6990dfbb..c483ea478 100644 --- a/gateway/assets/style.css +++ b/gateway/assets/style.css @@ -144,13 +144,22 @@ main section:not(:last-child) { border-bottom: 1px solid var(--dark-white); } +main section header { + background-color: var(--near-white); +} + .grid { display: grid; + overflow-x: auto; +} + +.grid .grid { + overflow-x: visible; } .grid > div { padding: .7em; - border-top: 1px solid var(--dark-white); + border-bottom: 1px solid var(--dark-white); } .grid.dir { @@ -165,8 +174,8 @@ main section:not(:last-child) { padding-right: 1em; } -.grid.dir > div:nth-child(-n+4) { - border-top: 0; +.grid.dir > div:nth-last-child(-n+4) { + border-bottom: 0; } .grid.dir > div:nth-of-type(8n+5), @@ -176,6 +185,31 @@ main section:not(:last-child) { background-color: var(--near-white); } +.grid.dag { + grid-template-columns: max-content 1fr; +} + +.grid.dag .grid { + padding: 0; +} + +.grid.dag > div:nth-last-child(-n+2) { + border-bottom: 0; +} + +.grid.dag > div { + background: white +} + +.grid.dag > div:nth-child(4n), +.grid.dag > div:nth-child(4n+3) { + background-color: var(--near-white); +} + +section > .grid.dag > div:nth-of-type(2n+1) { + padding-left: 1em; +} + .type-icon, .type-icon > * { width: 1.15em @@ -222,4 +256,3 @@ main section:not(:last-child) { display: none; } } - diff --git a/gateway/assets/templates.go b/gateway/assets/templates.go index 73b4696ae..a7c98c221 100644 --- a/gateway/assets/templates.go +++ b/gateway/assets/templates.go @@ -24,9 +24,15 @@ func iconFromExt(filename string) string { return "ipfs-_blank" // Default is blank icon. } +// args is a helper function to allow sending more than one object to a template. +func args(args ...interface{}) []interface{} { + return args +} + var funcMap = template.FuncMap{ "iconFromExt": iconFromExt, "urlEscape": urlEscape, + "args": args, } func readFile(fs fs.FS, filename string) ([]byte, error) { diff --git a/gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block b/gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block new file mode 100644 index 000000000..db8145d6d Binary files /dev/null and b/gateway/assets/test/dag/bafyreiagdtlc3xwhbeywzpwmxvwkogcujhlsm6f4cfdgpjpyu77gkubro4.block differ diff --git a/gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block b/gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block new file mode 100644 index 000000000..5605a83bd Binary files /dev/null and b/gateway/assets/test/dag/bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq.block differ diff --git a/gateway/assets/test/dag/bafyreifqscjzoqx6rs6niehqo4jcsc63ihej2hxnskywhs7bx5t3prhpdu.block b/gateway/assets/test/dag/bafyreifqscjzoqx6rs6niehqo4jcsc63ihej2hxnskywhs7bx5t3prhpdu.block new file mode 100644 index 000000000..0c38bd6fb Binary files /dev/null and b/gateway/assets/test/dag/bafyreifqscjzoqx6rs6niehqo4jcsc63ihej2hxnskywhs7bx5t3prhpdu.block differ diff --git a/gateway/assets/test/dag/bafyreihnpl7ami7esahkfdnemm6idx4r2n6u3apmtcrxlqwuapgjsciihy.block b/gateway/assets/test/dag/bafyreihnpl7ami7esahkfdnemm6idx4r2n6u3apmtcrxlqwuapgjsciihy.block new file mode 100644 index 000000000..e38c2c2aa Binary files /dev/null and b/gateway/assets/test/dag/bafyreihnpl7ami7esahkfdnemm6idx4r2n6u3apmtcrxlqwuapgjsciihy.block differ diff --git a/gateway/assets/test/main.go b/gateway/assets/test/main.go index 5041a9c71..6aab99c68 100644 --- a/gateway/assets/test/main.go +++ b/gateway/assets/test/main.go @@ -1,14 +1,30 @@ package main import ( + "embed" "fmt" "net/http" "os" "strconv" + "strings" "github.com/ipfs/boxo/gateway/assets" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/node/basicnode" + mc "github.com/multiformats/go-multicodec" + + // Ensure basic codecs are registered. + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" + _ "github.com/ipld/go-ipld-prime/codec/raw" ) +//go:embed dag/*.block +var embeds embed.FS + const ( testPath = "/ipfs/QmFooBarQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7/a/b/c" ) @@ -19,9 +35,9 @@ var directoryTestData = assets.DirectoryTemplateData{ URL: "http://example.com", Title: "Support", }}, + GatewayURL: "//localhost:3000", + DNSLink: true, }, - GatewayURL: "//localhost:3000", - DNSLink: true, Listing: []assets.DirectoryItem{{ Size: "25 MiB", Name: "short-film.mov", @@ -62,17 +78,61 @@ var directoryTestData = assets.DirectoryTemplateData{ Hash: "QmFooBazBar2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7", } -var dagTestData = assets.DagTemplateData{ - GlobalData: assets.GlobalData{ - Menu: []assets.MenuItem{{ - URL: "http://example.com", - Title: "Support", - }}, - }, - Path: "/ipfs/baguqeerabn4wonmz6icnk7dfckuizcsf4e4igua2ohdboecku225xxmujepa", - CID: "baguqeerabn4wonmz6icnk7dfckuizcsf4e4igua2ohdboecku225xxmujepa", - CodecName: "dag-json", - CodecHex: "0x129", +var dagTestData = map[string]*assets.DagTemplateData{} + +func loadDagTestData() { + entries, err := embeds.ReadDir("dag") + if err != nil { + panic(err) + } + + for _, entry := range entries { + cidStr := strings.TrimSuffix(entry.Name(), ".block") + cid, err := cid.Decode(cidStr) + if err != nil { + panic(err) + } + + f, err := embeds.Open("dag/" + entry.Name()) + if err != nil { + panic(err) + } + + codec := cid.Prefix().Codec + decoder, err := multicodec.LookupDecoder(codec) + if err != nil { + panic(err) + } + + node := basicnode.Prototype.Any.NewBuilder() + err = decoder(node, f) + if err != nil { + panic(err) + } + + cidCodec := mc.Code(cid.Prefix().Codec) + + dag, err := assets.ParseNode(node.Build()) + if err != nil { + panic(err) + } + + dagTestData[cid.String()] = &assets.DagTemplateData{ + GlobalData: assets.GlobalData{ + Menu: []assets.MenuItem{{ + URL: "http://example.com", + Title: "Support", + }}, + GatewayURL: "//localhost:3000", + DNSLink: true, + }, + Path: "/ipfs/" + cid.String(), + CID: cid.String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: dag, + } + } } func init() { @@ -86,6 +146,8 @@ func init() { ShortHash: "QmbW\u2026sMnR", }) } + + loadDagTestData() } func runTemplate(w http.ResponseWriter, filename string, data interface{}) { @@ -107,7 +169,12 @@ func main() { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/dag": - runTemplate(w, "dag.html", dagTestData) + cid := r.URL.Query().Get("cid") + if cid == "" { + cid = "bafyreifqscjzoqx6rs6niehqo4jcsc63ihej2hxnskywhs7bx5t3prhpdu" + } + + runTemplate(w, "dag.html", dagTestData[cid]) case "/directory": runTemplate(w, "directory.html", directoryTestData) case "/error": diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 6a7f37967..017e94662 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "html" "io" "net/http" "net/http/httptest" @@ -742,3 +743,28 @@ func TestIpnsTrustlessMode(t *testing.T) { doRequest(t, "/EmptyDir/", "trusted.com", http.StatusOK) doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest) } + +func TestDagJsonCborPreview(t *testing.T) { + ts, _, root := newTestServerAndNode(t, nil) + t.Logf("test server url: %s", ts.URL) + url := path.Join([]string{"/ipfs", root.String(), t.Name(), "example"}) + + t.Run("Strings Are Escaped", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, ts.URL+url, nil) + req.Header.Add("Accept", "text/html") + assert.NoError(t, err) + + res, err := doWithoutRedirect(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + assert.NoError(t, err) + + script := "window.alert('hacked')" + escaped := html.EscapeString(script) + + assert.Contains(t, string(body), escaped) + assert.NotContains(t, string(body), script) + }) +} diff --git a/gateway/handler.go b/gateway/handler.go index 9d81e7922..240dc25cb 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -17,6 +17,7 @@ import ( "time" ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/gateway/assets" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p/core/peer" @@ -908,3 +909,25 @@ func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Requ return true } + +// getTemplateGlobalData returns the global data necessary by most templates. +func (i *handler) getTemplateGlobalData(r *http.Request, contentPath ipath.Path) assets.GlobalData { + // gatewayURL is used to link to other root CIDs. THis will be blank unless + // subdomain or DNSLink resolution is being used for this request. + var gatewayURL string + if h, ok := r.Context().Value(SubdomainHostnameKey).(string); ok { + gatewayURL = "//" + h + } else if h, ok := r.Context().Value(DNSLinkHostnameKey).(string); ok { + gatewayURL = "//" + h + } else { + gatewayURL = "" + } + + dnsLink := assets.HasDNSLinkOrigin(gatewayURL, contentPath.String()) + + return assets.GlobalData{ + Menu: i.config.Menu, + GatewayURL: gatewayURL, + DNSLink: dnsLink, + } +} diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 6c2a47ec9..b25570790 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -120,7 +120,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt download := r.URL.Query().Get("download") == "true" if isDAG && acceptsHTML && !download { - return i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) + return i.serveCodecHTML(ctx, w, r, blockCid, blockData, resolvedPath, contentPath) } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. @@ -152,7 +152,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return i.serveCodecConverted(ctx, w, r, blockCid, blockData, contentPath, toCodec, modtime, begin) } -func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { +func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { // A HTML directory index will be presented, be sure to set the correct // type instead of relying on autodetection (which may fail). w.Header().Set("Content-Type", "text/html") @@ -171,13 +171,12 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) if err := assets.DagTemplate.Execute(w, assets.DagTemplateData{ - GlobalData: assets.GlobalData{ - Menu: i.config.Menu, - }, - Path: contentPath.String(), - CID: resolvedPath.Cid().String(), - CodecName: cidCodec.String(), - CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + GlobalData: i.getTemplateGlobalData(r, contentPath), + Path: contentPath.String(), + CID: resolvedPath.Cid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: parseNode(blockCid, blockData), }); err != nil { err = fmt.Errorf("failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw: %w", err) i.webError(w, r, err, http.StatusInternalServerError) @@ -187,6 +186,30 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * return true } +// parseNode does a best effort attempt to parse this request's block such that +// a preview can be displayed in the gateway. If something fails along the way, +// returns nil, therefore not displaying the preview. +func parseNode(blockCid cid.Cid, blockData io.ReadSeekCloser) *assets.ParsedNode { + codec := blockCid.Prefix().Codec + decoder, err := multicodec.LookupDecoder(codec) + if err != nil { + return nil + } + + nodeBuilder := basicnode.Prototype.Any.NewBuilder() + err = decoder(nodeBuilder, blockData) + if err != nil { + return nil + } + + parsedNode, err := assets.ParseNode(nodeBuilder.Build()) + if err != nil { + return nil + } + + return parsedNode +} + // serveCodecRaw returns the raw block without any conversion func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockData io.ReadSeekCloser, contentPath ipath.Path, name string, modtime, begin time.Time) bool { // ServeContent will take care of diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 893755d05..82c2a297e 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -178,41 +178,21 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } size := humanize.Bytes(directoryMetadata.dagSize) - hash := resolvedPath.Cid().String() - - // Gateway root URL to be used when linking to other rootIDs. - // This will be blank unless subdomain or DNSLink resolution is being used - // for this request. - var gwURL string - - // Ensure correct URL in DNSLink and Subdomain Gateways. - if h, ok := r.Context().Value(SubdomainHostnameKey).(string); ok { - gwURL = "//" + h - } else if h, ok := r.Context().Value(DNSLinkHostnameKey).(string); ok { - gwURL = "//" + h - } else { - gwURL = "" - } - - dnslink := assets.HasDNSLinkOrigin(gwURL, contentPath.String()) + globalData := i.getTemplateGlobalData(r, contentPath) // See comment above where originalUrlPath is declared. tplData := assets.DirectoryTemplateData{ - GlobalData: assets.GlobalData{ - Menu: i.config.Menu, - }, - GatewayURL: gwURL, - DNSLink: dnslink, + GlobalData: globalData, Listing: dirListing, Size: size, Path: contentPath.String(), - Breadcrumbs: assets.Breadcrumbs(contentPath.String(), dnslink), + Breadcrumbs: assets.Breadcrumbs(contentPath.String(), globalData.DNSLink), BackLink: backLink, Hash: hash, } - logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) + logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) if err := assets.DirectoryTemplate.Execute(w, tplData); err != nil { i.webError(w, r, err, http.StatusInternalServerError) diff --git a/gateway/testdata/fixtures.car b/gateway/testdata/fixtures.car index e01ca5c31..06cf1c44d 100644 Binary files a/gateway/testdata/fixtures.car and b/gateway/testdata/fixtures.car differ