fix: use %-encoded headers in most compatible way

client/rpc/api.go

@@ -1,18 +1,24 @@
 package rpc
 
 import (
+	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
+	"github.com/blang/semver/v4"
 	iface "github.com/ipfs/boxo/coreiface"
 	caopts "github.com/ipfs/boxo/coreiface/options"
 	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/go-cid"
 	legacy "github.com/ipfs/go-ipld-legacy"
+	ipfs "github.com/ipfs/kubo"
 	dagpb "github.com/ipld/go-codec-dagpb"
 	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -42,6 +48,8 @@ type HttpApi struct {
 	Headers     http.Header
 	applyGlobal func(*requestBuilder)
 	ipldDecoder *legacy.Decoder
+	versionOnce sync.Once
+	version     *semver.Version
 }
 
 // NewLocalApi tries to construct new HttpApi instance communicating with local
@@ -151,6 +159,7 @@ func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) {
 	api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
 		return fmt.Errorf("unexpected redirect")
 	}
+
 	return api, nil
 }
 
@@ -160,14 +169,19 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
 		return nil, err
 	}
 
-	subApi := *api
-	subApi.applyGlobal = func(req *requestBuilder) {
-		if options.Offline {
-			req.Option("offline", options.Offline)
-		}
+	subApi := &HttpApi{
+		url:     api.url,
+		httpcli: api.httpcli,
+		Headers: api.Headers,
+		applyGlobal: func(req *requestBuilder) {
+			if options.Offline {
+				req.Option("offline", options.Offline)
+			}
+		},
+		ipldDecoder: api.ipldDecoder,
 	}
 
-	return &subApi, nil
+	return subApi, nil
 }
 
 func (api *HttpApi) Request(command string, args ...string) RequestBuilder {
@@ -228,3 +242,35 @@ func (api *HttpApi) PubSub() iface.PubSubAPI {
 func (api *HttpApi) Routing() iface.RoutingAPI {
 	return (*RoutingAPI)(api)
 }
+
+func (api *HttpApi) loadRemoteVersion() error {
+	if api.version == nil {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
+		defer cancel()
+
+		resp, err := api.Request("version").Send(ctx)
+		if err != nil {
+			return err
+		}
+		if resp.Error != nil {
+			return resp.Error
+		}
+		defer resp.Close()
+		var out ipfs.VersionInfo
+		dec := json.NewDecoder(resp.Output)
+		if err := dec.Decode(&out); err != nil {
+			return err
+		}
+
+		remoteVersion, err := semver.New(out.Version)
+		if err != nil {
+			return err
+		}
+
+		api.versionOnce.Do(func() {
+			api.version = remoteVersion
+		})
+	}
+
+	return nil
+}

client/rpc/requestbuilder.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/ipfs/boxo/files"
+	ipfs "github.com/ipfs/kubo"
 )
 
 type RequestBuilder interface {
@@ -25,11 +26,12 @@ type RequestBuilder interface {
 
 // requestBuilder is an IPFS commands request builder.
 type requestBuilder struct {
-	command string
-	args    []string
-	opts    map[string]string
-	headers map[string]string
-	body    io.Reader
+	command    string
+	args       []string
+	opts       map[string]string
+	headers    map[string]string
+	body       io.Reader
+	buildError error
 
 	shell *HttpApi
 }
@@ -60,7 +62,17 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder {
 func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder {
 	pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil)
 	d := files.NewMapDirectory(map[string]files.Node{"": pr})
-	r.body = files.NewMultiFileReader(d, false)
+
+	err := r.shell.loadRemoteVersion()
+	if err != nil {
+		// Unfortunately, we cannot return an error here. Changing this API is also
+		// not the best since we would otherwise have an inconsistent RequestBuilder.
+		// We save the error and return it when calling [requestBuilder.Send].
+		r.buildError = err
+		return r
+	}
+
+	r.body = files.NewMultiFileReader(d, false, r.shell.version.LT(ipfs.EncodedAbsolutePathVersion))
 
 	return r
 }
@@ -97,6 +109,10 @@ func (r *requestBuilder) Header(name, value string) RequestBuilder {
 
 // Send sends the request and return the response.
 func (r *requestBuilder) Send(ctx context.Context) (*Response, error) {
+	if r.buildError != nil {
+		return nil, r.buildError
+	}
+
 	r.shell.applyGlobal(r)
 
 	req := NewRequest(ctx, r.shell.url, r.command, r.args...)

client/rpc/unixfs.go

@@ -14,6 +14,7 @@ import (
 	unixfs "github.com/ipfs/boxo/ipld/unixfs"
 	unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb"
 	"github.com/ipfs/go-cid"
+	ipfs "github.com/ipfs/kubo"
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -62,7 +63,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	}
 
 	d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side
-	req.Body(files.NewMultiFileReader(d, false))
+
+	err = api.core().loadRemoteVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	req.Body(files.NewMultiFileReader(d, false, api.version.LT(ipfs.EncodedAbsolutePathVersion)))
 
 	var out addEvent
 	resp, err := req.Send(ctx)

cmd/ipfs/main.go

@@ -2,22 +2,27 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
 	"time"
 
+	"github.com/blang/semver/v4"
 	"github.com/google/uuid"
 	u "github.com/ipfs/boxo/util"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/go-ipfs-cmds/cli"
 	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
 	logging "github.com/ipfs/go-log"
+	ipfs "github.com/ipfs/kubo"
 	"github.com/ipfs/kubo/cmd/ipfs/util"
 	oldcmds "github.com/ipfs/kubo/commands"
 	"github.com/ipfs/kubo/core"
@@ -313,9 +318,18 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
 	default:
 		return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
 	}
-	opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{
+
+	httpClient := &http.Client{
 		Transport: otelhttp.NewTransport(tpt),
-	}))
+	}
+	opts = append(opts, cmdhttp.ClientWithHTTPClient(httpClient))
+
+	// Fetch remove version, as some feature compatibility might change depending on it.
+	remoteVersion, err := getRemoteVersion(tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)})
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, cmdhttp.ClientWithRawAbsPath(remoteVersion.LT(ipfs.EncodedAbsolutePathVersion)))
 
 	return tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)}, nil
 }
@@ -415,3 +429,40 @@ func resolveAddr(ctx context.Context, addr ma.Multiaddr) (ma.Multiaddr, error) {
 
 	return addrs[0], nil
 }
+
+type nopWriter struct {
+	io.Writer
+}
+
+func (nw nopWriter) Close() error {
+	return nil
+}
+
+func getRemoteVersion(exe cmds.Executor) (*semver.Version, error) {
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
+	defer cancel()
+
+	req, err := cmds.NewRequest(ctx, []string{"version"}, nil, nil, nil, Root)
+	if err != nil {
+		return nil, err
+	}
+
+	var buf bytes.Buffer
+	re, err := cmds.NewWriterResponseEmitter(nopWriter{&buf}, req)
+	if err != nil {
+		return nil, err
+	}
+
+	err = exe.Execute(req, re, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var out ipfs.VersionInfo
+	dec := json.NewDecoder(&buf)
+	if err := dec.Decode(&out); err != nil {
+		return nil, err
+	}
+
+	return semver.New(out.Version)
+}

docs/changelogs/v0.23.md

@@ -32,6 +32,20 @@ $ ipfs config --json Swarm.Transports.Multiplexers.Mplex 200
 We will completely remove Mplex in v0.24 as it makes protocols very bad to implement,
 if you are in this situation you need to add yamux support to your other implementation.
 
+#### Binary characters in file names: no longer works with old clients and new Kubo servers
+
+In this version, we updated Kubo to support Go 1.20+. In Go 1.20, a regression
+regarding multipart headers was [introduced](https://github.com/golang/go/issues/60674).
+This only affects when a file name has binary characters in its name. As a consequence,
+we had to update the encoding of the file name headers. This is the compatibility
+table:
+
+|            | New Client  | Old Client                     |
+|------------|-------------|--------------------------------|
+| New Server | ✅          | ❌ (loud fail w/ binary paths) |
+| Old Server | ✅          | ✅                             |
+
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.20
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f
+	github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.29.2
 	github.com/multiformats/go-multiaddr v0.10.1

docs/examples/kubo-as-a-library/go.sum

@@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f h1:sXqGLIATCsBdHse7S9n6e328NhORvVM64+4IRuFlpmI=
-github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
+github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5 h1:yH0AHLdnI8c1Jdtp/GN2W9UWRxuQ6prfClZyCiGoTHo=
+github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f
+	github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0
@@ -27,7 +27,7 @@ require (
 	github.com/ipfs/go-ds-measure v0.2.0
 	github.com/ipfs/go-fs-lock v0.0.7
 	github.com/ipfs/go-graphsync v0.14.4
-	github.com/ipfs/go-ipfs-cmds v0.9.0
+	github.com/ipfs/go-ipfs-cmds v0.9.1-0.20230817103510-c4268a17faa6
 	github.com/ipfs/go-ipld-format v0.5.0
 	github.com/ipfs/go-ipld-git v0.1.1
 	github.com/ipfs/go-ipld-legacy v0.2.1
@@ -92,7 +92,6 @@ require (
 require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/Jorropo/jsync v1.0.1 // indirect
-	github.com/Kubuxu/go-os-helper v0.0.1 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -166,7 +165,7 @@ require (
 	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
 	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
-	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

go.sum

@@ -49,8 +49,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
 github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
-github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
-github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@@ -337,8 +335,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f h1:sXqGLIATCsBdHse7S9n6e328NhORvVM64+4IRuFlpmI=
-github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
+github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5 h1:yH0AHLdnI8c1Jdtp/GN2W9UWRxuQ6prfClZyCiGoTHo=
+github.com/ipfs/boxo v0.11.1-0.20230817103630-f4838887b6e5/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
@@ -383,8 +381,8 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X
 github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
-github.com/ipfs/go-ipfs-cmds v0.9.0 h1:K0VcXg1l1k6aY6sHnoxYcyimyJQbcV1ueXuWgThmK9Q=
-github.com/ipfs/go-ipfs-cmds v0.9.0/go.mod h1:SBFHK8WNwC416QWH9Vz1Ql42SSMAOqKpaHUMBu3jpLo=
+github.com/ipfs/go-ipfs-cmds v0.9.1-0.20230817103510-c4268a17faa6 h1:WU2SpQhBc8yzjG5cpcDjYguEtbDpLVXjLKOqW1CAM3Y=
+github.com/ipfs/go-ipfs-cmds v0.9.1-0.20230817103510-c4268a17faa6/go.mod h1:OMMX/4WtqvDh8VbIxWLVAWH6HkVy2WMjnsEzZZcgsdw=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
@@ -572,12 +570,14 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8
 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=