diff --git a/assets/assets.go b/assets/assets.go index 0c58feb3a2c..f6968dc2738 100644 --- a/assets/assets.go +++ b/assets/assets.go @@ -80,13 +80,9 @@ func addAssetList(nd *core.IpfsNode, l []string) (cid.Cid, error) { return cid.Cid{}, err } - if err := nd.Pinning.Pin(nd.Context(), dir, true); err != nil { + if err := nd.Pinning.Pin(nd.Context(), "assets", dir, true); err != nil { return cid.Cid{}, fmt.Errorf("assets: Pinning on init-docu failed: %s", err) } - if err := nd.Pinning.Flush(); err != nil { - return cid.Cid{}, fmt.Errorf("assets: Pinning flush failed: %s", err) - } - return dir.Cid(), nil } diff --git a/core/builder.go b/core/builder.go index 97b105a3ad2..15113748db0 100644 --- a/core/builder.go +++ b/core/builder.go @@ -268,15 +268,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.Blocks = bserv.New(n.Blockstore, n.Exchange) n.DAG = dag.NewDAGService(n.Blocks) - internalDag := dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore))) - n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG, internalDag) - if err != nil { - // TODO: we should move towards only running 'NewPinner' explicitly on - // node init instead of implicitly here as a result of the pinner keys - // not being found in the datastore. - // this is kinda sketchy and could cause data loss - n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG, internalDag) - } + n.Pinning = pin.NewPinner(n.DAG, n.Repo.Datastore()) n.Resolver = resolver.NewBasicResolver(n.DAG) if cfg.Online { diff --git a/core/commands/add.go b/core/commands/add.go index c3399835d59..f8dba17d80c 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -39,6 +39,7 @@ const ( onlyHashOptionName = "only-hash" chunkerOptionName = "chunker" pinOptionName = "pin" + pinPathOptionName = "pinpath" rawLeavesOptionName = "raw-leaves" noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" @@ -119,6 +120,7 @@ You can now check what blocks have been created by: cmdkit.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."), cmdkit.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").WithDefault("size-262144"), cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + cmdkit.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"), cmdkit.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), cmdkit.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), @@ -160,6 +162,7 @@ You can now check what blocks have been created by: silent, _ := req.Options[silentOptionName].(bool) chunker, _ := req.Options[chunkerOptionName].(string) dopin, _ := req.Options[pinOptionName].(bool) + pinPath, _ := req.Options[pinPathOptionName].(string) rawblks, rbset := req.Options[rawLeavesOptionName].(bool) nocopy, _ := req.Options[noCopyOptionName].(bool) fscache, _ := req.Options[fstoreCacheOptionName].(bool) @@ -185,6 +188,7 @@ You can now check what blocks have been created by: options.Unixfs.Chunker(chunker), options.Unixfs.Pin(dopin), + options.Unixfs.PinPath(pinPath), options.Unixfs.HashOnly(hash), options.Unixfs.FsCache(fscache), options.Unixfs.Nocopy(nocopy), diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 73c3999a00a..da155c43312 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,7 +7,6 @@ import ( "github.com/ipfs/go-ipfs/core/commands/cmdenv" "github.com/ipfs/go-ipfs/core/coredag" - "github.com/ipfs/go-ipfs/pin" path "gx/ipfs/QmNYPETsdAu2uQ1k9q9S1jYEGURaLHV6cbYRSVFVRftpF8/go-path" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" @@ -60,7 +59,7 @@ into an object of the specified format. Options: []cmdkit.Option{ cmdkit.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), cmdkit.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), - cmdkit.BoolOption("pin", "Pin this object when adding."), + cmdkit.StringOption("pin", "Pin this object when adding.").WithDefault(""), cmdkit.StringOption("hash", "Hash function to use").WithDefault(""), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -72,7 +71,7 @@ into an object of the specified format. ienc, _ := req.Options["input-enc"].(string) format, _ := req.Options["format"].(string) hash, _ := req.Options["hash"].(string) - dopin, _ := req.Options["pin"].(bool) + pinpath, _ := req.Options["pin"].(string) // mhType tells inputParser which hash should be used. MaxUint64 means 'use // default hash' (sha256 for cbor, sha1 for git..) @@ -89,10 +88,6 @@ into an object of the specified format. cids := cid.NewSet() b := ipld.NewBatch(req.Context, nd.DAG) - if dopin { - defer nd.Blockstore.PinLock().Unlock() - } - it := req.Files.Entries() for it.Next() { file := files.FileFromEntry(it) @@ -128,13 +123,11 @@ into an object of the specified format. return err } - if dopin { - cids.ForEach(func(c cid.Cid) error { - nd.Pinning.PinWithMode(c, pin.Recursive) - return nil + if pinpath != "" { + err := cids.ForEach(func(c cid.Cid) error { + return nd.Pinning.AddPin(pinpath, c, true) }) - err := nd.Pinning.Flush() if err != nil { return err } diff --git a/core/commands/object/object.go b/core/commands/object/object.go index cf1d232740b..b8ad30b648a 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -383,6 +383,7 @@ And then run: cmdkit.StringOption("inputenc", "Encoding type of input data. One of: {\"protobuf\", \"json\"}.").WithDefault("json"), cmdkit.StringOption("datafieldenc", "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"), cmdkit.BoolOption("pin", "Pin this object when adding."), + cmdkit.StringOption("pinpath", "Pin under this path").WithDefault("added/"), cmdkit.BoolOption("quiet", "q", "Write minimal output."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -406,7 +407,12 @@ And then run: return err } - dopin, _ := req.Options["pin"].(bool) + pinpath, _ := req.Options["pinpath"].(string) + if err != nil { + return err + } + + pin, _ := req.Options["pin"].(bool) if err != nil { return err } @@ -414,7 +420,8 @@ And then run: p, err := api.Object().Put(req.Context, file, options.Object.DataType(datafieldenc), options.Object.InputEnc(inputenc), - options.Object.Pin(dopin)) + options.Object.Pin(pin), + options.Object.PinPath(pinpath)) if err != nil { return err } diff --git a/core/commands/pin.go b/core/commands/pin.go index b84fa3d3d33..258f00a9f8f 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -11,7 +11,6 @@ import ( cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" e "github.com/ipfs/go-ipfs/core/commands/e" iface "github.com/ipfs/go-ipfs/core/coreapi/interface" - options "github.com/ipfs/go-ipfs/core/coreapi/interface/options" corerepo "github.com/ipfs/go-ipfs/core/corerepo" pin "github.com/ipfs/go-ipfs/pin" @@ -59,11 +58,12 @@ var addPinCmd = &cmds.Command{ }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), + cmdkit.StringArg("ipfs-path", true, true, "Path(es) to object(s) to be pinned.").EnableStdin(), }, Options: []cmdkit.Option{ - cmdkit.BoolOption(pinRecursiveOptionName, "r", "Recursively pin the object linked to by the specified object(s).").WithDefault(true), + cmdkit.BoolOption("direct", "d", "Pin the object directly"), cmdkit.BoolOption(pinProgressOptionName, "Show progress"), + cmdkit.StringOption("pinpath", "P", "Pin path.").WithDefault("default/"), }, Type: AddPinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -77,18 +77,20 @@ var addPinCmd = &cmds.Command{ return err } - defer n.Blockstore.PinLock().Unlock() - - // set recursive flag - recursive, _ := req.Options[pinRecursiveOptionName].(bool) + // set direct flag + direct, _ := req.Options["direct"].(bool) showProgress, _ := req.Options[pinProgressOptionName].(bool) - if err := req.ParseBodyArgs(); err != nil { + if err = req.ParseBodyArgs(); err != nil { return err } + pinPath, _ := req.Options["pinpath"].(string) + + toPin := req.Arguments + if !showProgress { - added, err := corerepo.Pin(n.Pinning, api, req.Context, req.Arguments, recursive) + added, err := corerepo.Pin(n.Pinning, api, req.Context, pinPath, toPin, !direct) if err != nil { return err } @@ -105,7 +107,7 @@ var addPinCmd = &cmds.Command{ ch := make(chan pinResult, 1) go func() { - added, err := corerepo.Pin(n.Pinning, api, ctx, req.Arguments, recursive) + added, err := corerepo.Pin(n.Pinning, api, ctx, pinPath, toPin, !direct) ch <- pinResult{pins: added, err: err} }() @@ -137,9 +139,10 @@ var addPinCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddPinOutput) error { - rec, found := req.Options["recursive"].(bool) + direct, found := req.Options["direct"].(bool) + var pintype string - if rec || !found { + if !direct || !found { pintype = "recursively" } else { pintype = "directly" @@ -186,16 +189,15 @@ var rmPinCmd = &cmds.Command{ Tagline: "Remove pinned objects from local storage.", ShortDescription: ` Removes the pin from the given object allowing it to be garbage -collected if needed. (By default, recursively. Use -r=false for direct pins.) +collected if needed. By default, removes recursive pins. `, }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("ipfs-path", true, true, "Path to object(s) to be unpinned.").EnableStdin(), + cmdkit.StringArg("pin-path", true, true, "Pin paths").EnableStdin(), }, Options: []cmdkit.Option{ - cmdkit.BoolOption(pinRecursiveOptionName, "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), - cmdkit.BoolOption("explain", "e", "Check for other pinned objects which could cause specified object(s) to be indirectly pinned").WithDefault(false), + cmdkit.BoolOption("direct", "d", "Unpins a direct pin").WithDefault(false), }, Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -210,23 +212,19 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) } // set recursive flag - recursive, _ := req.Options[pinRecursiveOptionName].(bool) + direct, _ := req.Options["direct"].(bool) + recursive := !direct if err := req.ParseBodyArgs(); err != nil { return err - - explain, _, err := req.Option("explain").Bool() - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return } - removed, err := corerepo.Unpin(n.Pinning, api, req.Context, req.Arguments, recursive, explain) + removed, err := corerepo.Unpin(n.Pinning, api, req.Context, req.Arguments, recursive) if err != nil { return err } - return cmds.EmitOnce(res, &PinOutput{cidsToStrings(removed)}) + return cmds.EmitOnce(res, &PinOutput{removed}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { @@ -288,10 +286,12 @@ Example: }, Arguments: []cmdkit.Argument{ + // TODO: implement path listing cmdkit.StringArg("ipfs-path", false, true, "Path to object(s) to be listed."), }, Options: []cmdkit.Option{ cmdkit.StringOption(pinTypeOptionName, "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), + cmdkit.BoolOption("recursive", "r", "List all pins under given prefix"), cmdkit.BoolOption(pinQuietOptionName, "q", "Write just hashes of objects."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -300,15 +300,9 @@ Example: return err } - api, err := cmdenv.GetApi(env, req) - if err != nil { - return err - } - typeStr, _ := req.Options[pinTypeOptionName].(string) - if err != nil { - return err - } + + recursive, _ := req.Options["recursive"].(bool) switch typeStr { case "all", "direct", "indirect", "recursive": @@ -317,30 +311,29 @@ Example: return err } - var keys map[string]RefKeyObject - - if len(req.Arguments) > 0 { - keys, err = pinLsKeys(req.Context, req.Arguments, typeStr, n, api) - } else { - keys, err = pinLsAll(req.Context, typeStr, n) + prefix := "" + if len(req.Arguments) == 1 { + prefix = req.Arguments[0] } + keys, err := pinLsAll(req.Context, typeStr, prefix, recursive, n) + if err != nil { return err } - return cmds.EmitOnce(res, &RefKeyList{Keys: keys}) + return cmds.EmitOnce(res, keys) }, Type: RefKeyList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefKeyList) error { quiet, _ := req.Options[pinQuietOptionName].(bool) - for k, v := range out.Keys { + for _, v := range out.Keys { if quiet { - fmt.Fprintf(w, "%s\n", k) + fmt.Fprintf(w, "%s\n", v.Object.String()) } else { - fmt.Fprintf(w, "%s %s\n", k, v.Type) + fmt.Fprintf(w, "%s %s %s\n", v.Object.String(), v.PinType, v.PinPath) } } @@ -364,37 +357,30 @@ new pin and removing the old one. }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("from-path", true, false, "Path to old object."), + cmdkit.StringArg("pinpath", true, false, "Pin path of the old object."), cmdkit.StringArg("to-path", true, false, "Path to new object to be pinned."), }, - Options: []cmdkit.Option{ - cmdkit.BoolOption(pinUnpinOptionName, "Remove the old pin.").WithDefault(true), - }, - Type: PinOutput{}, + Options: []cmdkit.Option{}, + Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } - unpin, _ := req.Options[pinUnpinOptionName].(bool) - - from, err := iface.ParsePath(req.Arguments[0]) - if err != nil { - return err - } + from := req.Arguments[0] to, err := iface.ParsePath(req.Arguments[1]) if err != nil { return err } - err = api.Pin().Update(req.Context, from, to, options.Pin.Unpin(unpin)) + err = api.Pin().Update(req.Context, from, to) if err != nil { return err } - return cmds.EmitOnce(res, &PinOutput{Pins: []string{from.String(), to.String()}}) + return cmds.EmitOnce(res, &PinOutput{Pins: []string{from, to.String()}}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { @@ -454,22 +440,20 @@ var verifyPinCmd = &cmds.Command{ } type RefKeyObject struct { - Type string + PinType string + PinPath string + Object cid.Cid } type RefKeyList struct { - Keys map[string]RefKeyObject + Keys []RefKeyObject } -func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsNode, api iface.CoreAPI) (map[string]RefKeyObject, error) { - - mode, ok := pin.StringToMode(typeStr) - if !ok { - return nil, fmt.Errorf("invalid pin mode '%s'", typeStr) - } +func pinFindCids(args []string, ctx context.Context, api iface.CoreAPI, n *core.IpfsNode) (map[string]RefKeyObject, error) { keys := make(map[string]RefKeyObject) + cids := make([]cid.Cid, 0, len(args)) for _, p := range args { pth, err := iface.ParsePath(p) if err != nil { @@ -481,58 +465,100 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN return nil, err } - pinType, pinned, err := n.Pinning.IsPinnedWithType(c.Cid(), mode) - if err != nil { - return nil, err - } + cids = append(cids, c.Cid()) + } + + pinneds, err := n.Pinning.CheckIfPinned(cids...) + if err != nil { + return nil, err + } - if !pinned { + for _, p := range pinneds { + + if !p.Pinned() { return nil, fmt.Errorf("path '%s' is not pinned", p) } - switch pinType { - case "direct", "indirect", "recursive", "internal": - default: - pinType = "indirect through " + pinType - } - keys[c.Cid().String()] = RefKeyObject{ - Type: pinType, + keys[p.Key.String()] = RefKeyObject{ + PinType: p.String(), } } return keys, nil } -func pinLsAll(ctx context.Context, typeStr string, n *core.IpfsNode) (map[string]RefKeyObject, error) { +func pinLsAll(ctx context.Context, typeStr string, prefix string, recursive bool, n *core.IpfsNode) (*RefKeyList, error) { + // TODO: replace string with type? + keys := make([]RefKeyObject, 0) - keys := make(map[string]RefKeyObject) + var recursiveMap map[string]cid.Cid + var directMap map[string]cid.Cid - AddToResultKeys := func(keyList []cid.Cid, typeStr string) { - for _, c := range keyList { - keys[c.String()] = RefKeyObject{ - Type: typeStr, - } + if recursive { + var err error + recursiveMap, err = n.Pinning.PrefixedPins(prefix, true) + if err != nil { + return nil, err + } + directMap, err = n.Pinning.PrefixedPins(prefix, false) + if err != nil { + return nil, err + } + } else { + recursiveMap = make(map[string]cid.Cid) + c, err := n.Pinning.GetPin(prefix, true) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + recursiveMap[prefix] = *c + } + directMap = make(map[string]cid.Cid) + c, err = n.Pinning.GetPin(prefix, false) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + directMap[prefix] = *c } } + if typeStr == "recursive" || typeStr == "all" { + for path, c := range recursiveMap { + // TODO: review whole RefKeyObject thing + keys = append(keys, RefKeyObject{ + PinType: "recursive", + Object: c, + PinPath: path, + }) + } + } if typeStr == "direct" || typeStr == "all" { - AddToResultKeys(n.Pinning.DirectKeys(), "direct") + for path, c := range directMap { + // TODO: review whole RefKeyObject thing + keys = append(keys, RefKeyObject{ + PinType: "direct", + Object: c, + PinPath: path, + }) + } } if typeStr == "indirect" || typeStr == "all" { - set := cid.NewSet() - for _, k := range n.Pinning.RecursiveKeys() { - err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), k, set.Visit) + for p, c := range recursiveMap { + set := cid.NewSet() + err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), c, set.Visit) if err != nil { return nil, err } + for _, k := range set.Keys() { + keys = append(keys, RefKeyObject{ + PinType: "indirect", + Object: k, + PinPath: p, + }) + } } - AddToResultKeys(set.Keys(), "indirect") - } - if typeStr == "recursive" || typeStr == "all" { - AddToResultKeys(n.Pinning.RecursiveKeys(), "recursive") } - return keys, nil + return &RefKeyList{Keys: keys}, nil } // PinVerifyRes is the result returned for each pin checked in "pin verify" @@ -560,11 +586,19 @@ type pinVerifyOpts struct { func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts) <-chan interface{} { visited := make(map[string]PinStatus) - bs := n.Blocks.Blockstore() DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := dag.GetLinksWithDAG(DAG) - recPins := n.Pinning.RecursiveKeys() + + recPins, err := n.Pinning.PinnedCids(true) + if err != nil { + status := PinStatus{Ok: false} + // TODO: do something about this error reporting + status.BadNodes = []BadNode{BadNode{Err: err.Error()}} + out := make(chan interface{}) + out <- &PinVerifyRes{"", status} + return out + } var checkPin func(root cid.Cid) PinStatus checkPin = func(root cid.Cid) PinStatus { diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index c988affb091..58fc400efa0 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -7,7 +7,6 @@ import ( cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" filestore "github.com/ipfs/go-ipfs/filestore" - pin "github.com/ipfs/go-ipfs/pin" balanced "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/importer/balanced" ihelper "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/importer/helpers" @@ -48,6 +47,7 @@ time. Options: []cmdkit.Option{ cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + cmdkit.StringOption("pinpath", "P", "Pin object under this path.").WithDefault("added/"), }, Arguments: []cmdkit.Argument{ cmdkit.StringArg("url", true, false, "URL to add to IPFS"), @@ -90,11 +90,6 @@ time. return fmt.Errorf("expected code 200, got: %d", hres.StatusCode) } - if dopin { - // Take the pinlock - defer n.Blockstore.PinLock().Unlock() - } - chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize) prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256) dbp := &ihelper.DagBuilderParams{ @@ -116,10 +111,11 @@ time. return err } + pinPath, _ := req.Options["pinpath"].(string) + c := root.Cid() if dopin { - n.Pinning.PinWithMode(c, pin.Recursive) - if err := n.Pinning.Flush(); err != nil { + if err := n.Pinning.AddPin(pinPath, c, true); err != nil { return err } } diff --git a/core/coreapi/interface/options/object.go b/core/coreapi/interface/options/object.go index e484a9f3632..3847e3a64f7 100644 --- a/core/coreapi/interface/options/object.go +++ b/core/coreapi/interface/options/object.go @@ -8,6 +8,7 @@ type ObjectPutSettings struct { InputEnc string DataType string Pin bool + PinPath string } type ObjectAddLinkSettings struct { @@ -37,6 +38,7 @@ func ObjectPutOptions(opts ...ObjectPutOption) (*ObjectPutSettings, error) { InputEnc: "json", DataType: "text", Pin: false, + PinPath: "added/", } for _, opt := range opts { @@ -114,6 +116,14 @@ func (objectOpts) Pin(pin bool) ObjectPutOption { } } +// PinPath is an option for Object.Put which specifies under which path to pin the object, default is "added/" +func (objectOpts) PinPath(pinPath string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.PinPath = pinPath + return nil + } +} + // Create is an option for Object.AddLink which specifies whether create required // directories for the child func (objectOpts) Create(create bool) ObjectAddLinkOption { diff --git a/core/coreapi/interface/options/pin.go b/core/coreapi/interface/options/pin.go index 9d1107f927d..e5ddb7ce5ba 100644 --- a/core/coreapi/interface/options/pin.go +++ b/core/coreapi/interface/options/pin.go @@ -2,14 +2,15 @@ package options type PinAddSettings struct { Recursive bool + PinPath string } type PinLsSettings struct { - Type string + Type string + Recursive bool } type PinUpdateSettings struct { - Unpin bool } type PinAddOption func(*PinAddSettings) error @@ -19,6 +20,7 @@ type PinUpdateOption func(*PinUpdateSettings) error func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { options := &PinAddSettings{ Recursive: true, + PinPath: "default/", } for _, opt := range opts { @@ -33,7 +35,8 @@ func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { options := &PinLsSettings{ - Type: "all", + Type: "all", + Recursive: false, } for _, opt := range opts { @@ -47,9 +50,7 @@ func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { } func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { - options := &PinUpdateSettings{ - Unpin: true, - } + options := &PinUpdateSettings{} for _, opt := range opts { err := opt(options) @@ -64,7 +65,8 @@ func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { type pinType struct{} type pinOpts struct { - Type pinType + Type pinType + Recursively bool } var Pin pinOpts @@ -75,24 +77,42 @@ func (pinType) All() PinLsOption { return Pin.pinType("all") } -// Recursive is an option for Pin.Ls which will make it only return recursive -// pins -func (pinType) Recursive() PinLsOption { - return Pin.pinType("recursive") -} - // Direct is an option for Pin.Ls which will make it only return direct (non // recursive) pins func (pinType) Direct() PinLsOption { return Pin.pinType("direct") } +// Direct is an option for Pin.Ls which will make it only return direct (non +// recursive) pins +func (pinType) Recursive() PinLsOption { + return Pin.pinType("recursive") +} + // Indirect is an option for Pin.Ls which will make it only return indirect pins // (objects referenced by other recursively pinned objects) func (pinType) Indirect() PinLsOption { return Pin.pinType("indirect") } +// Recursive is an option for Pin.Ls which allows to specify whether +// argument should be recursively searched for pins +func (pinOpts) RecursiveList(recursive bool) PinLsOption { + return func(settings *PinLsSettings) error { + settings.Recursive = recursive + return nil + } +} + +// PinPath is an option for Pin.Add which specifies pin path +// default: "default/" +func (pinOpts) PinPath(pinPath string) PinAddOption { + return func(settings *PinAddSettings) error { + settings.PinPath = pinPath + return nil + } +} + // Recursive is an option for Pin.Add which specifies whether to pin an entire // object tree or just one object. Default: true func (pinOpts) Recursive(recursive bool) PinAddOption { @@ -117,12 +137,3 @@ func (pinOpts) pinType(t string) PinLsOption { return nil } } - -// Unpin is an option for Pin.Update which specifies whether to remove the old pin. -// Default is true. -func (pinOpts) Unpin(unpin bool) PinUpdateOption { - return func(settings *PinUpdateSettings) error { - settings.Unpin = unpin - return nil - } -} diff --git a/core/coreapi/interface/options/unixfs.go b/core/coreapi/interface/options/unixfs.go index 5f92f3eea41..9bde2e799af 100644 --- a/core/coreapi/interface/options/unixfs.go +++ b/core/coreapi/interface/options/unixfs.go @@ -29,6 +29,7 @@ type UnixfsAddSettings struct { Layout Layout Pin bool + PinPath string OnlyHash bool FsCache bool NoCopy bool @@ -58,6 +59,7 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Layout: BalancedLayout, Pin: false, + PinPath: "added", OnlyHash: false, FsCache: false, NoCopy: false, @@ -209,6 +211,14 @@ func (unixfsOpts) Pin(pin bool) UnixfsAddOption { } } +// Pin tells the adder to pin the file root recursively after adding +func (unixfsOpts) PinPath(pinPath string) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.PinPath = pinPath + return nil + } +} + // HashOnly will make the adder calculate data hash without storing it in the // blockstore or announcing it to the network func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { diff --git a/core/coreapi/interface/pin.go b/core/coreapi/interface/pin.go index 2e119cbeae8..9152050a4d6 100644 --- a/core/coreapi/interface/pin.go +++ b/core/coreapi/interface/pin.go @@ -11,6 +11,9 @@ type Pin interface { // Path to the pinned object Path() ResolvedPath + // Pinpath of pinned object + PinPath() string + // Type of the pin Type() string } @@ -20,6 +23,9 @@ type PinStatus interface { // Ok indicates whether the pin has been verified to be correct Ok() bool + // Pinpath of pinned object + PinPath() string + // BadNodes returns any bad (usually missing) nodes from the pin BadNodes() []BadPinNode } @@ -37,17 +43,17 @@ type BadPinNode interface { type PinAPI interface { // Add creates new pin, be default recursive - pinning the whole referenced // tree - Add(context.Context, Path, ...options.PinAddOption) error + Add(context.Context, string, Path, ...options.PinAddOption) error // Ls returns list of pinned objects on this node - Ls(context.Context, ...options.PinLsOption) ([]Pin, error) + Ls(context.Context, string, ...options.PinLsOption) ([]Pin, error) // Rm removes pin for object specified by the path - Rm(context.Context, Path) error + Rm(context.Context, string) error // Update changes one pin to another, skipping checks for matching paths in // the old tree - Update(ctx context.Context, from Path, to Path, opts ...options.PinUpdateOption) error + Update(ctx context.Context, from string, to Path, opts ...options.PinUpdateOption) error // Verify verifies the integrity of pinned objects Verify(context.Context) (<-chan PinStatus, error) diff --git a/core/coreapi/interface/tests/pin.go b/core/coreapi/interface/tests/pin.go index 87ad8a00489..6fc5d90c655 100644 --- a/core/coreapi/interface/tests/pin.go +++ b/core/coreapi/interface/tests/pin.go @@ -9,6 +9,8 @@ import ( opt "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) +var testPrefix = "test/" + func (tp *provider) TestPin(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.Pin() == nil { @@ -35,7 +37,7 @@ func (tp *provider) TestPinAdd(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p) + err = api.Pin().Add(ctx, testPrefix, p) if err != nil { t.Error(err) } @@ -54,12 +56,12 @@ func (tp *provider) TestPinSimple(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p) + err = api.Pin().Add(ctx, testPrefix, p) if err != nil { t.Error(err) } - list, err := api.Pin().Ls(ctx) + list, err := api.Pin().Ls(ctx, testPrefix, opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -76,12 +78,12 @@ func (tp *provider) TestPinSimple(t *testing.T) { t.Error("unexpected pin type") } - err = api.Pin().Rm(ctx, p) + err = api.Pin().Rm(ctx, testPrefix+p.Cid().String()) if err != nil { t.Fatal(err) } - list, err = api.Pin().Ls(ctx) + list, err = api.Pin().Ls(ctx, testPrefix, opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -119,17 +121,17 @@ func (tp *provider) TestPinRecursive(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p2) + err = api.Pin().Add(ctx, testPrefix, p2) if err != nil { t.Error(err) } - err = api.Pin().Add(ctx, p3, opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, testPrefix, p3, opt.Pin.Recursive(false)) if err != nil { t.Error(err) } - list, err := api.Pin().Ls(ctx) + list, err := api.Pin().Ls(ctx, testPrefix, opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -138,7 +140,7 @@ func (tp *provider) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Direct()) + list, err = api.Pin().Ls(ctx, testPrefix, opt.Pin.Type.Direct(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -151,7 +153,7 @@ func (tp *provider) TestPinRecursive(t *testing.T) { t.Error("unexpected path") } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Recursive()) + list, err = api.Pin().Ls(ctx, testPrefix, opt.Pin.Type.Recursive(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -164,7 +166,7 @@ func (tp *provider) TestPinRecursive(t *testing.T) { t.Error("unexpected path") } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Indirect()) + list, err = api.Pin().Ls(ctx, testPrefix, opt.Pin.Type.Indirect(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } diff --git a/core/coreapi/interface/tests/unixfs.go b/core/coreapi/interface/tests/unixfs.go index e8a1aba32eb..7ab25b56d62 100644 --- a/core/coreapi/interface/tests/unixfs.go +++ b/core/coreapi/interface/tests/unixfs.go @@ -556,7 +556,7 @@ func (tp *provider) TestAddPinned(t *testing.T) { t.Error(err) } - pins, err := api.Pin().Ls(ctx) + pins, err := api.Pin().Ls(ctx, "", options.Pin.RecursiveList(true)) if len(pins) != 1 { t.Fatalf("expected 1 pin, got %d", len(pins)) } diff --git a/core/coreapi/object.go b/core/coreapi/object.go index 7fad4bd5925..26dd82ea905 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -14,7 +14,6 @@ import ( coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/dagutils" - "github.com/ipfs/go-ipfs/pin" ft "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" @@ -117,18 +116,13 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj return nil, err } - if options.Pin { - defer api.blockstore.PinLock().Unlock() - } - err = api.dag.Add(ctx, dagnode) if err != nil { return nil, err } if options.Pin { - api.pinning.PinWithMode(dagnode.Cid(), pin.Recursive) - err = api.pinning.Flush() + err = api.pinning.AddPin(options.PinPath, dagnode.Cid(), true) if err != nil { return nil, err } diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index da1050c233b..7b62f06b3e5 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -12,48 +12,50 @@ import ( cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" offline "gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline" + + "github.com/ipfs/go-ipfs/pin" ) type PinAPI CoreAPI -func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, pinpath string, p coreiface.Path, opts ...caopts.PinAddOption) error { + settings, err := caopts.PinAddOptions(opts...) if err != nil { return err } rp, err := api.core().ResolvePath(ctx, p) + if err != nil { return err } - defer api.blockstore.PinLock().Unlock() - - _, err = corerepo.Pin(api.pinning, api.core(), ctx, []string{rp.Cid().String()}, settings.Recursive) + _, err = corerepo.Pin(api.pinning, api.core(), ctx, pinpath, []string{rp.Cid().String()}, settings.Recursive) if err != nil { return err } - return api.pinning.Flush() + return nil } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, prefix string, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) { settings, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err } switch settings.Type { - case "all", "direct", "indirect", "recursive": + case "all", "direct", "recursive", "indirect": default: return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } - return api.pinLsAll(settings.Type, ctx) + return api.pinLsAll(settings.Type, prefix, ctx, settings.Recursive) } -func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { - _, err := corerepo.Unpin(api.pinning, api.core(), ctx, []string{p.String()}, true, true) +func (api *PinAPI) Rm(ctx context.Context, p string) error { + _, err := corerepo.Unpin(api.pinning, api.core(), ctx, []string{p}, true) if err != nil { return err } @@ -61,35 +63,17 @@ func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { return nil } -func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path, opts ...caopts.PinUpdateOption) error { - settings, err := caopts.PinUpdateOptions(opts...) - if err != nil { - return err - } - - fp, err := api.core().ResolvePath(ctx, from) - if err != nil { - return err - } - +func (api *PinAPI) Update(ctx context.Context, from string, to coreiface.Path, _ ...caopts.PinUpdateOption) error { tp, err := api.core().ResolvePath(ctx, to) if err != nil { return err } - - defer api.blockstore.PinLock().Unlock() - - err = api.pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin) - if err != nil { - return err - } - - return api.pinning.Flush() + return api.pinning.Update(ctx, from, tp.Cid()) } type pinStatus struct { - cid cid.Cid ok bool + pinPath string badNodes []coreiface.BadPinNode } @@ -103,6 +87,10 @@ func (s *pinStatus) Ok() bool { return s.ok } +func (s *pinStatus) PinPath() string { + return s.pinPath +} + func (s *pinStatus) BadNodes() []coreiface.BadPinNode { return s.badNodes } @@ -120,25 +108,30 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro bs := api.blockstore DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := merkledag.GetLinksWithDAG(DAG) - recPins := api.pinning.RecursiveKeys() + recPins, err := api.pinning.PrefixedPins("", true) + + if err != nil { + return nil, err + } - var checkPin func(root cid.Cid) *pinStatus - checkPin = func(root cid.Cid) *pinStatus { - if status, ok := visited[root]; ok { + var checkPin func(root cid.Cid, pinPath string) *pinStatus + checkPin = func(root cid.Cid, pinPath string) *pinStatus { + status, ok := visited[root] + if ok { return status } links, err := getLinks(ctx, root) if err != nil { - status := &pinStatus{ok: false, cid: root} + status := &pinStatus{ok: false, pinPath: pinPath} status.badNodes = []coreiface.BadPinNode{&badNode{path: coreiface.IpldPath(root), err: err}} visited[root] = status return status } - status := &pinStatus{ok: true, cid: root} + status = &pinStatus{ok: true} for _, lnk := range links { - res := checkPin(lnk.Cid) + res := checkPin(lnk.Cid, pinPath) if !res.ok { status.ok = false status.badNodes = append(status.badNodes, res.badNodes...) @@ -152,8 +145,8 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro out := make(chan coreiface.PinStatus) go func() { defer close(out) - for _, c := range recPins { - out <- checkPin(c) + for path, c := range recPins { + out <- checkPin(c, path) } }() @@ -162,9 +155,14 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro type pinInfo struct { pinType string + pinPath string path coreiface.ResolvedPath } +func (p *pinInfo) PinPath() string { + return p.pinPath +} + func (p *pinInfo) Path() coreiface.ResolvedPath { return p.path } @@ -173,42 +171,75 @@ func (p *pinInfo) Type() string { return p.pinType } -func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pin, error) { +func (api *PinAPI) pinLsAll(typeStr string, prefix string, ctx context.Context, recursive bool) ([]coreiface.Pin, error) { + keys := make([]coreiface.Pin, 0) - keys := make(map[cid.Cid]*pinInfo) + var recursiveMap map[string]cid.Cid + var directMap map[string]cid.Cid - AddToResultKeys := func(keyList []cid.Cid, typeStr string) { - for _, c := range keyList { - keys[c] = &pinInfo{ - pinType: typeStr, - path: coreiface.IpldPath(c), - } + if recursive { + var err error + recursiveMap, err = api.pinning.PrefixedPins(prefix, true) + if err != nil { + return nil, err + } + directMap, err = api.pinning.PrefixedPins(prefix, false) + if err != nil { + return nil, err + } + } else { + recursiveMap = make(map[string]cid.Cid) + c, err := api.pinning.GetPin(prefix, true) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + recursiveMap[prefix] = *c + } + directMap = make(map[string]cid.Cid) + c, err = api.pinning.GetPin(prefix, false) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + directMap[prefix] = *c } } + if typeStr == "recursive" || typeStr == "all" { + for pinPath, c := range recursiveMap { + keys = append(keys, &pinInfo{ + pinType: "recursive", + path: coreiface.IpldPath(c), + pinPath: pinPath, + }) + } + } if typeStr == "direct" || typeStr == "all" { - AddToResultKeys(api.pinning.DirectKeys(), "direct") + for pinPath, c := range directMap { + keys = append(keys, &pinInfo{ + pinType: "direct", + path: coreiface.IpldPath(c), + pinPath: pinPath, + }) + } } if typeStr == "indirect" || typeStr == "all" { - set := cid.NewSet() - for _, k := range api.pinning.RecursiveKeys() { - err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.dag), k, set.Visit) + for pinPath, c := range recursiveMap { + set := cid.NewSet() + err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.dag), c, set.Visit) if err != nil { return nil, err } + for _, k := range set.Keys() { + keys = append(keys, &pinInfo{ + pinType: "indirect", + pinPath: pinPath, + path: coreiface.IpldPath(k), + }) + } } - AddToResultKeys(set.Keys(), "indirect") - } - if typeStr == "recursive" || typeStr == "all" { - AddToResultKeys(api.pinning.RecursiveKeys(), "recursive") } - out := make([]coreiface.Pin, 0, len(keys)) - for _, v := range keys { - out = append(out, v) - } - - return out, nil + return keys, nil } func (api *PinAPI) core() coreiface.CoreAPI { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 68af2d52e3c..c4157e72f86 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -87,6 +87,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.Hidden = settings.Hidden fileAdder.Wrap = settings.Wrap fileAdder.Pin = settings.Pin && !settings.OnlyHash + fileAdder.PinPath = settings.PinPath fileAdder.Silent = settings.Silent fileAdder.RawLeaves = settings.RawLeaves fileAdder.NoCopy = settings.NoCopy diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index bfa1cf549f0..4442b489dc0 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -16,14 +16,14 @@ package corerepo import ( "context" "fmt" - "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/pin" "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" ) -func Pin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, paths []string, recursive bool) ([]cid.Cid, error) { +func Pin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, pinpath string, paths []string, recursive bool) ([]cid.Cid, error) { out := make([]cid.Cid, len(paths)) for i, fpath := range paths { @@ -36,22 +36,17 @@ func Pin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, paths []str if err != nil { return nil, fmt.Errorf("pin: %s", err) } - err = pinning.Pin(ctx, dagnode, recursive) + err = pinning.Pin(ctx, pinpath, dagnode, recursive) if err != nil { return nil, fmt.Errorf("pin: %s", err) } out[i] = dagnode.Cid() } - err := pinning.Flush() - if err != nil { - return nil, err - } - return out, nil } -func Unpin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, paths []string, recursive bool, explain bool) ([]cid.Cid, error) { +func UnpinPaths(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, paths []string, recursive bool) ([]cid.Cid, error) { unpinned := make([]cid.Cid, len(paths)) for i, p := range paths { @@ -65,16 +60,26 @@ func Unpin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, paths []s return nil, err } - err = pinning.Unpin(ctx, k.Cid(), recursive, explain) + err = pinning.UnpinCid(k.Cid(), recursive) if err != nil { return nil, err } unpinned[i] = k.Cid() } - err := pinning.Flush() - if err != nil { - return nil, err + return unpinned, nil +} + +func Unpin(pinning pin.Pinner, api iface.CoreAPI, ctx context.Context, pinPaths []string, recursive bool) ([]string, error) { + unpinned := make([]string, len(pinPaths)) + + for i, p := range pinPaths { + err := pinning.Unpin(p, recursive) + if err != nil { + return nil, err + } + unpinned[i] = p } + return unpinned, nil } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 8ce2a608ab6..e5a1048ed65 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -71,6 +71,7 @@ type Adder struct { Progress bool Hidden bool Pin bool + PinPath string Trickle bool RawLeaves bool Silent bool @@ -179,15 +180,14 @@ func (adder *Adder) PinRoot() error { } if adder.tempRoot.Defined() { - err := adder.pinning.Unpin(adder.ctx, adder.tempRoot, true, false) + err := adder.pinning.UnpinCidUnderPrefix("tmp/temproot/", adder.tempRoot, true) if err != nil { return err } adder.tempRoot = rnk } - adder.pinning.PinWithMode(rnk, pin.Recursive) - return adder.pinning.Flush() + return adder.pinning.AddPin(adder.PinPath, rnk, true) } // Finalize flushes the mfs root directory and returns the mfs root node. diff --git a/exchange/reprovide/providers.go b/exchange/reprovide/providers.go index a1ccad71a17..be448684959 100644 --- a/exchange/reprovide/providers.go +++ b/exchange/reprovide/providers.go @@ -47,16 +47,26 @@ func NewPinnedProvider(pinning pin.Pinner, dag ipld.DAGService, onlyRoots bool) func pinSet(ctx context.Context, pinning pin.Pinner, dag ipld.DAGService, onlyRoots bool) (*cidutil.StreamingSet, error) { set := cidutil.NewStreamingSet() + recursiveKeys, err := pinning.PinnedCids(true) + if err != nil { + return nil, err + } + + directKeys, err := pinning.PinnedCids(false) + if err != nil { + return nil, err + } + go func() { ctx, cancel := context.WithCancel(ctx) defer cancel() defer close(set.New) - for _, key := range pinning.DirectKeys() { + for _, key := range directKeys { set.Visitor(ctx)(key) } - for _, key := range pinning.RecursiveKeys() { + for _, key := range recursiveKeys { set.Visitor(ctx)(key) if !onlyRoots { diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index 146ae94039e..dbb4e7317b6 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -18,16 +18,6 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - err := n.Pinning.Pin(ctx, emptyDir, false) - if err != nil { - return err - } - - err = n.Pinning.Flush() - if err != nil { - return err - } - pub := nsys.NewIpnsPublisher(n.Routing, n.Repo.Datastore()) return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) diff --git a/namesys/publisher.go b/namesys/publisher.go index 63cae54ce37..3b500b285e1 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -291,18 +291,6 @@ func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec func InitializeKeyspace(ctx context.Context, pub Publisher, pins pin.Pinner, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - // pin recursively because this might already be pinned - // and doing a direct pin would throw an error in that case - err := pins.Pin(ctx, emptyDir, true) - if err != nil { - return err - } - - err = pins.Flush() - if err != nil { - return err - } - return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) } diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 1c5c6a7af15..19a7d41fb28 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -198,13 +198,19 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo } return links, nil } - err := Descendants(ctx, getLinks, gcs, pn.RecursiveKeys()) + recursiveKeys, err := pn.PinnedCids(true) if err != nil { errors = true - select { - case output <- Result{Error: err}: - case <-ctx.Done(): - return nil, ctx.Err() + output <- Result{Error: err} + } else { + err := Descendants(ctx, getLinks, gcs, recursiveKeys) + if err != nil { + errors = true + select { + case output <- Result{Error: err}: + case <-ctx.Done(): + return nil, ctx.Err() + } } } @@ -230,11 +236,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo } } - for _, k := range pn.DirectKeys() { - gcs.Add(k) - } - - err = Descendants(ctx, getLinks, gcs, pn.InternalPins()) + directKeys, err := pn.PinnedCids(false) if err != nil { errors = true select { @@ -244,6 +246,10 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo } } + for _, k := range directKeys { + gcs.Add(k) + } + if errors { return nil, ErrCannotFetchAllLinks } diff --git a/pin/pin.go b/pin/pin.go index 85195d5d22e..f68d4ab20e1 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -5,43 +5,20 @@ package pin import ( "context" "fmt" - "os" - "sync" - "time" - "github.com/ipfs/go-ipfs/dagutils" + dutils "github.com/ipfs/go-ipfs/dagutils" mdag "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" logging "gx/ipfs/QmcuXC5cxs79ro2cUuHs4HQ2bkDLJUYokwL8aivcX6HW3C/go-log" ds "gx/ipfs/Qmf4xQhNomPNhrtZc67qSnfJSjxjXs9LWvknJtSXwimPrM/go-datastore" + dsq "gx/ipfs/Qmf4xQhNomPNhrtZc67qSnfJSjxjXs9LWvknJtSXwimPrM/go-datastore/query" ) var log = logging.Logger("pin") -var pinDatastoreKey = ds.NewKey("/local/pins") - -var emptyKey cid.Cid - -func init() { - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) - } - emptyKey = e -} - -const ( - linkRecursive = "recursive" - linkDirect = "direct" - linkIndirect = "indirect" - linkInternal = "internal" - linkNotPinned = "not pinned" - linkAny = "any" - linkAll = "all" -) +var pinDatastoreKey = "/local/pins" // Mode allows to specify different types of pin (recursive, direct etc.). // See the Pin Modes constants for a full list. @@ -58,46 +35,10 @@ const ( // Indirect pins are cids who have some ancestor pinned recursively. Indirect - // Internal pins are cids used to keep the internal state of the pinner. - Internal - // NotPinned NotPinned - - // Any refers to any pinned cid - Any ) -// ModeToString returns a human-readable name for the Mode. -func ModeToString(mode Mode) (string, bool) { - m := map[Mode]string{ - Recursive: linkRecursive, - Direct: linkDirect, - Indirect: linkIndirect, - Internal: linkInternal, - NotPinned: linkNotPinned, - Any: linkAny, - } - s, ok := m[mode] - return s, ok -} - -// StringToMode parses the result of ModeToString() back to a Mode. -// It returns a boolean which is set to false if the mode is unknown. -func StringToMode(s string) (Mode, bool) { - m := map[string]Mode{ - linkRecursive: Recursive, - linkDirect: Direct, - linkIndirect: Indirect, - linkInternal: Internal, - linkNotPinned: NotPinned, - linkAny: Any, - linkAll: Any, // "all" and "any" means the same thing - } - mode, ok := m[s] - return mode, ok -} - // A Pinner provides the necessary methods to keep track of Nodes which are // to be kept locally, according to a pin mode. In practice, a Pinner is in // in charge of keeping the list of items from the local storage that should @@ -107,48 +48,41 @@ type Pinner interface { // and an explanation of why its pinned IsPinned(cid.Cid) (string, bool, error) - // IsPinnedWithType returns whether or not the given cid is pinned with the - // given pin type, as well as returning the type of pin its pinned with. - IsPinnedWithType(cid.Cid, Mode) (string, bool, error) + // GetPin returns cid pinned at given path + GetPin(path string, recursive bool) (*cid.Cid, error) // Pin the given node, optionally recursively. - Pin(ctx context.Context, node ipld.Node, recursive bool) error + Pin(ctx context.Context, path string, node ipld.Node, recursive bool) error + + // Unpin the given path with given mode. + Unpin(path string, recursive bool) error + + // Unpin the given cid with given mode. + UnpinCid(cid cid.Cid, recursive bool) error - // Unpin the given cid. If recursive is true, removes either a recursive or - // a direct pin. If recursive is false, only removes a direct pin. - Unpin(ctx context.Context, cid cid.Cid, recursive bool, explain bool) error + UnpinCidUnderPrefix(path string, c cid.Cid, recursive bool) error // Update updates a recursive pin from one cid to another // this is more efficient than simply pinning the new one and unpinning the // old one - Update(ctx context.Context, from, to cid.Cid, unpin bool) error + Update(ctx context.Context, path string, to cid.Cid) error + + // Check if given cid is pinned with given mode, return pin path + IsPinPresent(cid cid.Cid, recursive bool) (string, error) // Check if a set of keys are pinned, more efficient than // calling IsPinned for each key CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) - // PinWithMode is for manually editing the pin structure. Use with + // AddPin is for manually editing the pin structure. Use with // care! If used improperly, garbage collection may not be // successful. - PinWithMode(cid.Cid, Mode) - - // RemovePinWithMode is for manually editing the pin structure. - // Use with care! If used improperly, garbage collection may not - // be successful. - RemovePinWithMode(cid.Cid, Mode) - - // Flush writes the pin state to the backing datastore - Flush() error + AddPin(path string, cid cid.Cid, recursive bool) error - // DirectKeys returns all directly pinned cids - DirectKeys() []cid.Cid + // PinnedCids returns all pinned cids (recursive or direct) + PinnedCids(recursive bool) ([]cid.Cid, error) - // DirectKeys returns all recursively pinned cids - RecursiveKeys() []cid.Cid - - // InternalPins returns all cids kept pinned for the internal state of the - // pinner - InternalPins() []cid.Cid + PrefixedPins(prefix string, recursive bool) (map[string]cid.Cid, error) } // Pinned represents CID which has been pinned with a pinning strategy. @@ -158,6 +92,7 @@ type Pinner interface { type Pinned struct { Key cid.Cid Mode Mode + Path string Via cid.Cid } @@ -168,51 +103,44 @@ func (p Pinned) Pinned() bool { // String Returns pin status as string func (p Pinned) String() string { + var result string switch p.Mode { case NotPinned: - return "not pinned" + result = "not pinned" case Indirect: - return fmt.Sprintf("pinned via %s", p.Via) - default: - modeStr, _ := ModeToString(p.Mode) - return fmt.Sprintf("pinned: %s", modeStr) + result = fmt.Sprintf("pinned under %s via %s", p.Path, p.Via) + case Recursive: + result = fmt.Sprintf("pinned under %s recursively", p.Path) + case Direct: + result = fmt.Sprintf("pinned under %s directly", p.Path) } + return result } // pinner implements the Pinner interface type pinner struct { - lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - - // Track the keys used for storing the pinning state, so gc does - // not delete them. - internalPin *cid.Set - dserv ipld.DAGService - internal ipld.DAGService // dagservice used to store internal objects - dstore ds.Datastore + dserv ipld.DAGService + dstore ds.Datastore } // NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() +func NewPinner(dserv ipld.DAGService, dstore ds.Datastore) Pinner { + return &pinner{dserv: dserv, dstore: dstore} +} - return &pinner{ - recursePin: rcset, - directPin: dirset, - dserv: serv, - dstore: dstore, - internal: internal, - internalPin: cid.NewSet(), +// Pin the given node, optionally recursive +// If path ends with /, cid will be appended to path. +func (p *pinner) AddPin(path string, c cid.Cid, recursive bool) error { + if len(path) == 0 || path[len(path)-1] == '/' { + path += c.String() } + return p.dstore.Put(ds.NewKey(pathToDSKey(path, recursive)), c.Bytes()) } // Pin the given node, optionally recursive -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { - p.lock.Lock() - defer p.lock.Unlock() +// Also fetches its data +// If path ends with /, cid will be appended to path. +func (p *pinner) Pin(ctx context.Context, path string, node ipld.Node, recurse bool) error { err := p.dserv.Add(ctx, node) if err != nil { return err @@ -221,195 +149,135 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { c := node.Cid() if recurse { - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - p.lock.Unlock() // fetch entire graph err := mdag.FetchGraph(ctx, c, p.dserv) - p.lock.Lock() if err != nil { return err } - - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - - p.recursePin.Add(c) } else { - p.lock.Unlock() _, err := p.dserv.Get(ctx, c) - p.lock.Lock() if err != nil { return err } + } - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) - } + return p.AddPin(path, c, recurse) +} - p.directPin.Add(c) +func (p *pinner) GetPin(path string, recursive bool) (*cid.Cid, error) { + bytes, err := p.dstore.Get(ds.NewKey(pathToDSKey(path, recursive))) + if err == ds.ErrNotFound { + return nil, ErrNotPinned + } else if err != nil { + return nil, err + } + + cid, err := cid.Cast(bytes) + if err != nil { + return nil, err } - return nil + return &cid, nil } -// ErrNotPinned is returned when trying to unpin items which are not pinned. var ErrNotPinned = fmt.Errorf("not pinned") // Unpin a given key -func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool, explain bool) error { - p.lock.Lock() - defer p.lock.Unlock() - - var pinMode Mode - if recursive { - pinMode = Recursive - } else { - pinMode = Direct - } - _, pinned, err := p.isPinnedWithType(c, pinMode) +func (p *pinner) Unpin(path string, recursive bool) error { + _, err := p.GetPin(path, recursive) if err != nil { return err } - if pinned { - if recursive { - p.recursePin.Remove(c) - return nil - } else { - p.directPin.Remove(c) - return nil - } - } else if recursive { - _, pinned, err := p.isPinnedWithType(c, Direct) - if err != nil { - return err - } - if pinned { - p.directPin.Remove(c) - return nil - } - } - - if explain { - reason, pinned, err := p.isPinnedWithType(c, Any) - if err != nil { - return err - } - if !pinned { - return ErrNotPinned - } - switch reason { - case "recursive": - return fmt.Errorf("%s is pinned recursively", c) - default: - return fmt.Errorf("%s is pinned indirectly under %s", c, reason) - } - } else if recursive { - return fmt.Errorf("%s is not pinned recursively or directly", c) - } else { - return fmt.Errorf("%s is not pinned directly", c) - } - -} -func (p *pinner) isInternalPin(c cid.Cid) bool { - return p.internalPin.Has(c) + return p.dstore.Delete(ds.NewKey(pathToDSKey(path, recursive))) } -// IsPinned returns whether or not the given key is pinned -// and an explanation of why its pinned -func (p *pinner) IsPinned(c cid.Cid) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(c, Any) -} +// Unpin a given cid pinned under given prefix +// e.g UCUP("/temp/", QmAFobaz... will unpin /temp/QmAFobaz +// If path does not end in a slash, just acts as Unpin() +func (p *pinner) UnpinCidUnderPrefix(path string, c cid.Cid, recursive bool) error { + if path[len(path)-1] == '/' { + path += c.String() + } + _, err := p.GetPin(path, recursive) + if err != nil { + return err + } -// IsPinnedWithType returns whether or not the given cid is pinned with the -// given pin type, as well as returning the type of pin its pinned with. -func (p *pinner) IsPinnedWithType(c cid.Cid, mode Mode) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(c, mode) + return p.dstore.Delete(ds.NewKey(pathToDSKey(path, recursive))) } -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. -// intended for use by other pinned methods that already take locks -func (p *pinner) isPinnedWithType(c cid.Cid, mode Mode) (string, bool, error) { - switch mode { - case Any, Direct, Indirect, Recursive, Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, Direct, Indirect, Recursive, Internal, Any) - return "", false, err - } - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == Recursive { - return "", false, nil +func pinKeyPrefix(recursive bool) string { + prefix := "direct" + if recursive { + prefix = "recursive" } + return pinDatastoreKey + "/" + prefix + "/" +} +func pathToDSKey(path string, recursive bool) string { + return pinKeyPrefix(recursive) + path +} - if (mode == Direct || mode == Any) && p.directPin.Has(c) { - return linkDirect, true, nil +func (p *pinner) IsPinPresent(cid cid.Cid, recursive bool) (string, error) { + pinMap, err := p.PrefixedPins("", recursive) + if err != nil { + return "", err } - if mode == Direct { - return "", false, nil + for path, c := range pinMap { + if c == cid { + return path, nil + } } + return "", ErrNotPinned +} - if (mode == Internal || mode == Any) && p.isInternalPin(c) { - return linkInternal, true, nil - } - if mode == Internal { - return "", false, nil +func (p *pinner) IsPinned(cid cid.Cid) (string, bool, error) { + pinneds, err := p.CheckIfPinned(cid) + if err != nil { + return "", false, err } - - // Default is Indirect - visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err - } - if has { - return rc.String(), true, nil - } + if len(pinneds) != 1 { + return "", false, fmt.Errorf("CheckIfPinned returned %d results instead of 1", len(pinneds)) } - return "", false, nil + return pinneds[0].String(), pinneds[0].Pinned(), nil } // CheckIfPinned Checks if a set of keys are pinned, more efficient than // calling IsPinned for each key, returns the pinned status of cid(s) func (p *pinner) CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() + recursiveMap, err := p.PrefixedPins("", true) + if err != nil { + return nil, err + } + directMap, err := p.PrefixedPins("", false) + if err != nil { + return nil, err + } pinned := make([]Pinned, 0, len(cids)) toCheck := cid.NewSet() - // First check for non-Indirect pins directly for _, c := range cids { - if p.recursePin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Direct}) - } else if p.isInternalPin(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Internal}) - } else { - toCheck.Add(c) + toCheck.Visit(c) + } + + if toCheck.Len() == 0 { + return pinned, nil + } + + for path, c := range directMap { + if toCheck.Has(c) { + pinned = append(pinned, + Pinned{Key: c, Mode: Direct, Path: path}) + toCheck.Remove(c) } } + if toCheck.Len() == 0 { + return pinned, nil + } + // Now walk all recursive pins to check for indirect pins - var checkChildren func(cid.Cid, cid.Cid) error - checkChildren = func(rk, parentKey cid.Cid) error { + var checkChildren func(string, cid.Cid, cid.Cid) error + checkChildren = func(path string, rk, parentKey cid.Cid) error { links, err := ipld.GetLinks(context.TODO(), p.dserv, parentKey) if err != nil { return err @@ -419,11 +287,11 @@ func (p *pinner) CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) { if toCheck.Has(c) { pinned = append(pinned, - Pinned{Key: c, Mode: Indirect, Via: rk}) + Pinned{Key: c, Mode: Indirect, Via: rk, Path: path}) toCheck.Remove(c) } - err := checkChildren(rk, c) + err := checkChildren(path, rk, c) if err != nil { return err } @@ -435,8 +303,14 @@ func (p *pinner) CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) { return nil } - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) + for path, rk := range recursiveMap { + if toCheck.Has(rk) { + pinned = append(pinned, + Pinned{Key: rk, Mode: Recursive, Path: path}) + toCheck.Remove(rk) + } + + err := checkChildren(path, rk, rk) if err != nil { return nil, err } @@ -453,23 +327,6 @@ func (p *pinner) CheckIfPinned(cids ...cid.Cid) ([]Pinned, error) { return pinned, nil } -// RemovePinWithMode is for manually editing the pin structure. -// Use with care! If used improperly, garbage collection may not -// be successful. -func (p *pinner) RemovePinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Direct: - p.directPin.Remove(c) - case Recursive: - p.recursePin.Remove(c) - default: - // programmer error, panic OK - panic("unrecognized pin type") - } -} - func cidSetWithValues(cids []cid.Cid) *cid.Set { out := cid.NewSet() for _, c := range cids { @@ -478,168 +335,49 @@ func cidSetWithValues(cids []cid.Cid) *cid.Set { return out } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - p := new(pinner) +// PrefixedPins returns a map containing all pins of given kind under given prefix +func (p *pinner) PrefixedPins(prefix string, recursive bool) (map[string]cid.Cid, error) { + result, err := p.dstore.Query(dsq.Query{Prefix: pathToDSKey(prefix, recursive)}) - rootKey, err := d.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) if err != nil { return nil, err } - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) - defer cancel() - - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add + pinMap := make(map[string]cid.Cid) - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + for entry := range result.Next() { + c, err := cid.Cast(entry.Value) if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) + return nil, err } - p.recursePin = cidSetWithValues(recurseKeys) - } - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) + pinMap[entry.Key[len(pinKeyPrefix(recursive)):]] = c } - - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil -} - -// DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys() []cid.Cid { - return p.directPin.Keys() -} - -// RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys() []cid.Cid { - return p.recursePin.Keys() + return pinMap, nil } -// Update updates a recursive pin from one cid to another +// Update updates a pin from one cid to another // this is more efficient than simply pinning the new one and unpinning the // old one -func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { - p.lock.Lock() - defer p.lock.Unlock() - - if !p.recursePin.Has(from) { - return fmt.Errorf("'from' cid was not recursively pinned already") - } +func (p *pinner) Update(ctx context.Context, path string, to cid.Cid) error { - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) + c, err := p.GetPin(path, true) if err != nil { return err } - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush() error { - p.lock.Lock() - defer p.lock.Unlock() - - ctx := context.TODO() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } - } - - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) + err = dutils.DiffEnumerate(ctx, p.dserv, *c, to) if err != nil { return err } - err = p.internal.Add(ctx, root) + err = p.Unpin(path, true) if err != nil { return err } - k := root.Cid() + return p.AddPin(path, to, true) - internalset.Add(k) - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - p.internalPin = internalset - return nil -} - -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins() []cid.Cid { - p.lock.Lock() - defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out -} - -// PinWithMode allows the user to have fine grained control over pin -// counts -func (p *pinner) PinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Recursive: - p.recursePin.Add(c) - case Direct: - p.directPin.Add(c) - } } // hasChild recursively looks for a Cid among the children of a root Cid. @@ -667,3 +405,23 @@ func hasChild(ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Ci } return false, nil } + +func (p *pinner) PinnedCids(recursive bool) ([]cid.Cid, error) { + pinMap, err := p.PrefixedPins("", recursive) + if err != nil { + return nil, err + } + var cids []cid.Cid + for _, v := range pinMap { + cids = append(cids, v) + } + return cids, nil +} + +func (p *pinner) UnpinCid(cid cid.Cid, recursive bool) error { + path, err := p.IsPinPresent(cid, recursive) + if err != nil { + return err + } + return p.Unpin(path, recursive) +} diff --git a/pin/pin_test.go b/pin/pin_test.go index 542fe2b35ed..bd49c2d52aa 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -16,6 +16,8 @@ import ( dssync "gx/ipfs/Qmf4xQhNomPNhrtZc67qSnfJSjxjXs9LWvknJtSXwimPrM/go-datastore/sync" ) +var test_prefix = "test/" + var rand = util.NewTimeSeededRand() func randNode() (*mdag.ProtoNode, cid.Cid) { @@ -58,7 +60,7 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, ak := randNode() err := dserv.Add(ctx, a) @@ -67,7 +69,7 @@ func TestPinnerBasic(t *testing.T) { } // Pin A{} - err = p.Pin(ctx, a, false) + err = p.Pin(ctx, test_prefix, a, false) if err != nil { t.Fatal(err) } @@ -101,7 +103,7 @@ func TestPinnerBasic(t *testing.T) { bk := b.Cid() // recursively pin B{A,C} - err = p.Pin(ctx, b, true) + err = p.Pin(ctx, test_prefix, b, true) if err != nil { t.Fatal(err) } @@ -128,7 +130,7 @@ func TestPinnerBasic(t *testing.T) { } // Add D{A,C,E} - err = p.Pin(ctx, d, true) + err = p.Pin(ctx, test_prefix, d, true) if err != nil { t.Fatal(err) } @@ -137,20 +139,12 @@ func TestPinnerBasic(t *testing.T) { assertPinned(t, p, dk, "pinned node not found.") // Test recursive unpin - err = p.Unpin(ctx, dk, true, false) - if err != nil { - t.Fatal(err) - } - - err = p.Flush() + err = p.UnpinCidUnderPrefix(test_prefix, dk, true) if err != nil { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } + np := NewPinner(dserv, dstore) // Test directly pinned assertPinned(t, np, ak, "Could not find pinned node!") @@ -184,7 +178,7 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) aNodes := make([]*mdag.ProtoNode, aBranchLen) aKeys := make([]cid.Cid, aBranchLen) @@ -207,7 +201,7 @@ func TestIsPinnedLookup(t *testing.T) { } // Pin A5 recursively - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + if err := p.Pin(ctx, test_prefix, aNodes[aBranchLen-1], true); err != nil { t.Fatal(err) } @@ -245,13 +239,13 @@ func TestIsPinnedLookup(t *testing.T) { // Pin C recursively - if err := p.Pin(ctx, c, true); err != nil { + if err := p.Pin(ctx, test_prefix, c, true); err != nil { t.Fatal(err) } // Pin B recursively - if err := p.Pin(ctx, b, true); err != nil { + if err := p.Pin(ctx, test_prefix, b, true); err != nil { t.Fatal(err) } @@ -261,7 +255,7 @@ func TestIsPinnedLookup(t *testing.T) { assertPinned(t, p, bk, "B should be pinned") // Unpin A5 recursively - if err := p.Unpin(ctx, aKeys[5], true, false); err != nil { + if err := p.UnpinCidUnderPrefix(test_prefix, aKeys[5], true); err != nil { t.Fatal(err) } @@ -269,7 +263,7 @@ func TestIsPinnedLookup(t *testing.T) { assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") // Unpin B recursively - if err := p.Unpin(ctx, bk, true, false); err != nil { + if err := p.UnpinCidUnderPrefix(test_prefix, bk, true); err != nil { t.Fatal(err) } assertUnpinned(t, p, bk, "B should be unpinned") @@ -286,7 +280,7 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, _ := randNode() err := dserv.Add(ctx, a) @@ -295,38 +289,22 @@ func TestDuplicateSemantics(t *testing.T) { } // pin is recursively - err = p.Pin(ctx, a, true) + err = p.Pin(ctx, test_prefix, a, true) if err != nil { t.Fatal(err) } - // pinning directly should fail - err = p.Pin(ctx, a, false) - if err == nil { - t.Fatal("expected direct pin to fail") - } - - // pinning recursively again should succeed - err = p.Pin(ctx, a, true) + // pinning directly should succeed + err = p.Pin(ctx, test_prefix, a, false) if err != nil { t.Fatal(err) } -} -func TestFlush(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - _, k := randNode() - - p.PinWithMode(k, Recursive) - if err := p.Flush(); err != nil { + // pinning recursively again should still not fail + err = p.Pin(ctx, test_prefix, a, true) + if err != nil { t.Fatal(err) } - assertPinned(t, p, k, "expected key to still be pinned") } func TestPinRecursiveFail(t *testing.T) { @@ -336,7 +314,7 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, _ := randNode() b, _ := randNode() @@ -349,7 +327,7 @@ func TestPinRecursiveFail(t *testing.T) { mctx, cancel := context.WithTimeout(ctx, time.Millisecond) defer cancel() - err = p.Pin(mctx, a, true) + err = p.Pin(mctx, test_prefix, a, true) if err == nil { t.Fatal("should have failed to pin here") } @@ -367,7 +345,7 @@ func TestPinRecursiveFail(t *testing.T) { // this one is time based... but shouldnt cause any issues mctx, cancel = context.WithTimeout(ctx, time.Second) defer cancel() - err = p.Pin(mctx, a, true) + err = p.Pin(mctx, test_prefix, a, true) if err != nil { t.Fatal(err) } @@ -381,28 +359,28 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) n1, c1 := randNode() n2, c2 := randNode() dserv.Add(ctx, n1) dserv.Add(ctx, n2) - if err := p.Pin(ctx, n1, true); err != nil { + if err := p.Pin(ctx, test_prefix, n1, true); err != nil { t.Fatal(err) } - if err := p.Update(ctx, c1, c2, true); err != nil { + if err := p.Update(ctx, test_prefix+c1.String(), c2); err != nil { t.Fatal(err) } assertPinned(t, p, c2, "c2 should be pinned now") assertUnpinned(t, p, c1, "c1 should no longer be pinned") - if err := p.Update(ctx, c2, c1, false); err != nil { + if err := p.Update(ctx, test_prefix+c1.String(), c1); err != nil { t.Fatal(err) } - assertPinned(t, p, c2, "c2 should be pinned still") + assertUnpinned(t, p, c2, "c2 should no longer be pinned") assertPinned(t, p, c1, "c1 should be pinned now") } diff --git a/pin/set.go b/pin/set.go deleted file mode 100644 index ff7152685a7..00000000000 --- a/pin/set.go +++ /dev/null @@ -1,297 +0,0 @@ -package pin - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "hash/fnv" - "sort" - - "github.com/ipfs/go-ipfs/pin/internal/pb" - "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag" - - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" - "gx/ipfs/QmdxUuburamoF6zF9qjeQC4WYcWGbWuRmdLacMEsW8ioD8/gogo-protobuf/proto" -) - -const ( - // defaultFanout specifies the default number of fan-out links per layer - defaultFanout = 256 - - // maxItems is the maximum number of items that will fit in a single bucket - maxItems = 8192 -) - -func hash(seed uint32, c cid.Cid) uint32 { - var buf [4]byte - binary.LittleEndian.PutUint32(buf[:], seed) - h := fnv.New32a() - _, _ = h.Write(buf[:]) - _, _ = h.Write(c.Bytes()) - return h.Sum32() -} - -type itemIterator func() (c cid.Cid, ok bool) - -type keyObserver func(cid.Cid) - -type sortByHash struct { - links []*ipld.Link -} - -func (s sortByHash) Len() int { - return len(s.links) -} - -func (s sortByHash) Less(a, b int) bool { - return bytes.Compare(s.links[a].Cid.Bytes(), s.links[b].Cid.Bytes()) == -1 -} - -func (s sortByHash) Swap(a, b int) { - s.links[a], s.links[b] = s.links[b], s.links[a] -} - -func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - links := make([]*ipld.Link, 0, defaultFanout+maxItems) - for i := 0; i < defaultFanout; i++ { - links = append(links, &ipld.Link{Cid: emptyKey}) - } - - // add emptyKey to our set of internal pinset objects - n := &merkledag.ProtoNode{} - n.SetLinks(links) - - internalKeys(emptyKey) - - hdr := &pb.Set{ - Version: 1, - Fanout: defaultFanout, - Seed: depth, - } - if err := writeHdr(n, hdr); err != nil { - return nil, err - } - - if estimatedLen < maxItems { - // it'll probably fit - links := n.Links() - for i := 0; i < maxItems; i++ { - k, ok := iter() - if !ok { - // all done - break - } - - links = append(links, &ipld.Link{Cid: k}) - } - - n.SetLinks(links) - - // sort by hash, also swap item Data - s := sortByHash{ - links: n.Links()[defaultFanout:], - } - sort.Stable(s) - } - - hashed := make([][]cid.Cid, defaultFanout) - for { - // This loop essentially enumerates every single item in the set - // and maps them all into a set of buckets. Each bucket will be recursively - // turned into its own sub-set, and so on down the chain. Each sub-set - // gets added to the dagservice, and put into its place in a set nodes - // links array. - // - // Previously, the bucket was selected by taking an int32 from the hash of - // the input key + seed. This was erroneous as we would later be assigning - // the created sub-sets into an array of length 256 by the modulus of the - // int32 hash value with 256. This resulted in overwriting existing sub-sets - // and losing pins. The fix (a few lines down from this comment), is to - // map the hash value down to the 8 bit keyspace here while creating the - // buckets. This way, we avoid any overlapping later on. - k, ok := iter() - if !ok { - break - } - h := hash(depth, k) % defaultFanout - hashed[h] = append(hashed[h], k) - } - - for h, items := range hashed { - if len(items) == 0 { - // recursion base case - continue - } - - childIter := getCidListIterator(items) - - // recursively create a pinset from the items for this bucket index - child, err := storeItems(ctx, dag, uint64(len(items)), depth+1, childIter, internalKeys) - if err != nil { - return nil, err - } - - size, err := child.Size() - if err != nil { - return nil, err - } - - err = dag.Add(ctx, child) - if err != nil { - return nil, err - } - childKey := child.Cid() - - internalKeys(childKey) - - // overwrite the 'empty key' in the existing links array - n.Links()[h] = &ipld.Link{ - Cid: childKey, - Size: size, - } - } - return n, nil -} - -func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) { - hdrLenRaw, consumed := binary.Uvarint(n.Data()) - if consumed <= 0 { - return nil, errors.New("invalid Set header length") - } - - pbdata := n.Data()[consumed:] - if hdrLenRaw > uint64(len(pbdata)) { - return nil, errors.New("impossibly large Set header length") - } - // as hdrLenRaw was <= an int, we now know it fits in an int - hdrLen := int(hdrLenRaw) - var hdr pb.Set - if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil { - return nil, err - } - - if v := hdr.GetVersion(); v != 1 { - return nil, fmt.Errorf("unsupported Set version: %d", v) - } - if uint64(hdr.GetFanout()) > uint64(len(n.Links())) { - return nil, errors.New("impossibly large Fanout") - } - return &hdr, nil -} - -func writeHdr(n *merkledag.ProtoNode, hdr *pb.Set) error { - hdrData, err := proto.Marshal(hdr) - if err != nil { - return err - } - - // make enough space for the length prefix and the marshaled header data - data := make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)) - - // write the uvarint length of the header data - uvarlen := binary.PutUvarint(data, uint64(len(hdrData))) - - // append the actual protobuf data *after* the length value we wrote - data = append(data[:uvarlen], hdrData...) - - n.SetData(data) - return nil -} - -type walkerFunc func(idx int, link *ipld.Link) error - -func walkItems(ctx context.Context, dag ipld.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error { - hdr, err := readHdr(n) - if err != nil { - return err - } - // readHdr guarantees fanout is a safe value - fanout := hdr.GetFanout() - for i, l := range n.Links()[fanout:] { - if err := fn(i, l); err != nil { - return err - } - } - for _, l := range n.Links()[:fanout] { - c := l.Cid - children(c) - if c.Equals(emptyKey) { - continue - } - subtree, err := l.GetNode(ctx, dag) - if err != nil { - return err - } - - stpb, ok := subtree.(*merkledag.ProtoNode) - if !ok { - return merkledag.ErrNotProtobuf - } - - if err := walkItems(ctx, dag, stpb, fn, children); err != nil { - return err - } - } - return nil -} - -func loadSet(ctx context.Context, dag ipld.DAGService, root *merkledag.ProtoNode, name string, internalKeys keyObserver) ([]cid.Cid, error) { - l, err := root.GetNodeLink(name) - if err != nil { - return nil, err - } - - lnkc := l.Cid - internalKeys(lnkc) - - n, err := l.GetNode(ctx, dag) - if err != nil { - return nil, err - } - - pbn, ok := n.(*merkledag.ProtoNode) - if !ok { - return nil, merkledag.ErrNotProtobuf - } - - var res []cid.Cid - walk := func(idx int, link *ipld.Link) error { - res = append(res, link.Cid) - return nil - } - - if err := walkItems(ctx, dag, pbn, walk, internalKeys); err != nil { - return nil, err - } - return res, nil -} - -func getCidListIterator(cids []cid.Cid) itemIterator { - return func() (c cid.Cid, ok bool) { - if len(cids) == 0 { - return cid.Cid{}, false - } - - first := cids[0] - cids = cids[1:] - return first, true - } -} - -func storeSet(ctx context.Context, dag ipld.DAGService, cids []cid.Cid, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - iter := getCidListIterator(cids) - - n, err := storeItems(ctx, dag, uint64(len(cids)), 0, iter, internalKeys) - if err != nil { - return nil, err - } - err = dag.Add(ctx, n) - if err != nil { - return nil, err - } - internalKeys(n.Cid()) - return n, nil -} diff --git a/pin/set_test.go b/pin/set_test.go deleted file mode 100644 index 184041f3697..00000000000 --- a/pin/set_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package pin - -import ( - "context" - "encoding/binary" - "testing" - - dag "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag" - bserv "gx/ipfs/QmYPZzd9VqmJDwxUnThfeSbV1Y5o53aVPDijTB7j7rS9Ep/go-blockservice" - - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - blockstore "gx/ipfs/QmS2aqUZLJp8kF1ihE5rvDGE5LvmKDPnx32w9Z1BW9xLV5/go-ipfs-blockstore" - offline "gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline" - ds "gx/ipfs/Qmf4xQhNomPNhrtZc67qSnfJSjxjXs9LWvknJtSXwimPrM/go-datastore" - dsq "gx/ipfs/Qmf4xQhNomPNhrtZc67qSnfJSjxjXs9LWvknJtSXwimPrM/go-datastore/query" -) - -func ignoreCids(_ cid.Cid) {} - -func objCount(d ds.Datastore) int { - q := dsq.Query{KeysOnly: true} - res, err := d.Query(q) - if err != nil { - panic(err) - } - - var count int - for { - _, ok := res.NextSync() - if !ok { - break - } - - count++ - } - return count -} - -func TestSet(t *testing.T) { - dst := ds.NewMapDatastore() - bstore := blockstore.NewBlockstore(dst) - ds := dag.NewDAGService(bserv.New(bstore, offline.Exchange(bstore))) - - // this value triggers the creation of a recursive shard. - // If the recursive sharding is done improperly, this will result in - // an infinite recursion and crash (OOM) - limit := uint32((defaultFanout * maxItems) + 1) - - var inputs []cid.Cid - buf := make([]byte, 4) - for i := uint32(0); i < limit; i++ { - binary.BigEndian.PutUint32(buf, i) - c := dag.NewRawNode(buf).Cid() - inputs = append(inputs, c) - } - - _, err := storeSet(context.Background(), ds, inputs[:len(inputs)-1], ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs1 := objCount(dst) - - out, err := storeSet(context.Background(), ds, inputs, ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs2 := objCount(dst) - if objs2-objs1 > 2 { - t.Fatal("set sharding does not appear to be deterministic") - } - - // weird wrapper node because loadSet expects us to pass an - // object pointing to multiple named sets - setroot := &dag.ProtoNode{} - err = setroot.AddNodeLink("foo", out) - if err != nil { - t.Fatal(err) - } - - outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids) - if err != nil { - t.Fatal(err) - } - - if uint32(len(outset)) != limit { - t.Fatal("got wrong number", len(outset), limit) - } - - seen := cid.NewSet() - for _, c := range outset { - seen.Add(c) - } - - for _, c := range inputs { - if !seen.Has(c) { - t.Fatalf("expected to have '%s', didnt find it", c) - } - } -} diff --git a/test/sharness/t0010-basic-commands.sh b/test/sharness/t0010-basic-commands.sh index d5f89af33d9..fadd5f2cac9 100755 --- a/test/sharness/t0010-basic-commands.sh +++ b/test/sharness/t0010-basic-commands.sh @@ -127,7 +127,7 @@ test_expect_success "'ipfs commands --flags' succeeds" ' ' test_expect_success "'ipfs commands --flags' output looks good" ' - grep "ipfs pin add --recursive / ipfs pin add -r" commands.txt && + grep "ipfs pin add --direct / ipfs pin add -d" commands.txt && grep "ipfs id --format / ipfs id -f" commands.txt && grep "ipfs repo gc --quiet / ipfs repo gc -q" commands.txt ' diff --git a/test/sharness/t0025-datastores.sh b/test/sharness/t0025-datastores.sh index 21100e5ae31..2796c2f69a0 100755 --- a/test/sharness/t0025-datastores.sh +++ b/test/sharness/t0025-datastores.sh @@ -10,7 +10,7 @@ test_expect_success "'ipfs init --profile=badgerds' succeeds" ' ' test_expect_success "'ipfs pin ls' works" ' - ipfs pin ls | wc -l | grep 9 + ipfs pin ls -r | wc -l | grep 8 ' test_done diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index e16fb57c4ca..fc2c4395e8a 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -410,7 +410,7 @@ test_add_named_pipe() { err_prefix=$1 test_expect_success "useful error message when adding a named pipe" ' mkfifo named-pipe && - test_expect_code 1 ipfs add named-pipe 2>actual && + test_expect_code 1 ipfs add -P named-pipe named-pipe 2>actual && STAT=$(generic_stat named-pipe) && rm named-pipe && grep "Error: unrecognized file type for named-pipe: $STAT" actual && @@ -422,7 +422,7 @@ test_add_named_pipe() { mkdir -p named-pipe-dir && mkfifo named-pipe-dir/named-pipe && STAT=$(generic_stat named-pipe-dir/named-pipe) && - test_expect_code 1 ipfs add -r named-pipe-dir 2>actual && + test_expect_code 1 ipfs add -P named-pipe -r named-pipe-dir 2>actual && printf "Error:$err_prefix unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && rm named-pipe-dir/named-pipe && rmdir named-pipe-dir && @@ -709,7 +709,7 @@ test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYh # encoded with the blake2b-256 hash funtion test_add_cat_expensive '--hash=blake2b-256' "zDMZof1kwndounDzQCANUHjiE3zt1mPEgx7RE3JTHoZrRRa79xcv" -test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&progress=true&recursive=true&stream-channels=true:" +test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&pinpath=named-pipe&progress=true&recursive=true&stream-channels=true:" test_add_pwd_is_symlink diff --git a/test/sharness/t0046-id-hash.sh b/test/sharness/t0046-id-hash.sh index 27e49e855e1..3d49c01b2bc 100755 --- a/test/sharness/t0046-id-hash.sh +++ b/test/sharness/t0046-id-hash.sh @@ -34,7 +34,7 @@ test_expect_success "but can fetch it anyway" ' ' test_expect_success "block rm does nothing" ' - ipfs pin rm $HASH && + ipfs pin rm added/$HASH && ipfs block rm $HASH ' diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 17a71ae346a..83d424d6873 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -83,7 +83,6 @@ test_expect_success "add and pin directory" ' echo "file2" > adir/file2 && echo "file3" > adir/file3 && ipfs add -r adir - ipfs pin add -r $DIRHASH ' test_expect_success "can't remove pinned block" ' @@ -91,7 +90,7 @@ test_expect_success "can't remove pinned block" ' ' test_expect_success "can't remove pinned block: output looks good" ' - grep -q "$DIRHASH: pinned: recursive" block_rm_err + grep -q "$DIRHASH: pinned under added/$DIRHASH recursively" block_rm_err ' test_expect_success "can't remove indirectly pinned block" ' @@ -99,11 +98,11 @@ test_expect_success "can't remove indirectly pinned block" ' ' test_expect_success "can't remove indirectly pinned block: output looks good" ' - grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err + grep -q "$FILE1HASH: pinned under added/$DIRHASH via $DIRHASH" block_rm_err ' test_expect_success "remove pin" ' - ipfs pin rm -r $DIRHASH + ipfs pin rm added/$DIRHASH ' test_expect_success "multi-block 'ipfs block rm' succeeds" ' @@ -135,7 +134,7 @@ test_expect_success "'add some blocks' succeeds" ' test_expect_success "add and pin directory" ' ipfs add -r adir - ipfs pin add -r $DIRHASH + ipfs pin add $DIRHASH ' HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 3acf6c57ff9..a108b8311aa 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -21,7 +21,7 @@ test_expect_success "'ipfs add afile' succeeds" ' ' test_expect_success "added file was pinned" ' - ipfs pin ls --type=recursive >actual && + ipfs pin ls -r added/ --type=recursive >actual && grep "$HASH" actual ' @@ -40,11 +40,11 @@ test_expect_success "'ipfs repo gc' doesnt remove file" ' ' test_expect_success "'ipfs pin rm' succeeds" ' - ipfs pin rm -r "$HASH" >actual1 + ipfs pin rm "added/$HASH" >actual1 ' test_expect_success "'ipfs pin rm' output looks good" ' - echo "unpinned $HASH" >expected1 && + echo "unpinned added/$HASH" >expected1 && test_cmp expected1 actual1 ' @@ -54,7 +54,7 @@ test_expect_success "ipfs repo gc fully reverse ipfs add (part 1)" ' expected="$(directory_size "$IPFS_PATH/blocks")" && find "$IPFS_PATH/blocks" -type f && hash=$(ipfs add -q gcfile) && - ipfs pin rm -r $hash && + ipfs pin rm added/$hash && ipfs repo gc ' @@ -69,58 +69,36 @@ test_expect_success "ipfs repo gc fully reverse ipfs add (part 2)" ' test_launch_ipfs_daemon --offline test_expect_success "file no longer pinned" ' - ipfs pin ls --type=recursive --quiet >actual2 && + ipfs pin ls -r --type=recursive --quiet >actual2 && test_expect_code 1 grep $HASH actual2 ' test_expect_success "recursively pin afile(default action)" ' - HASH=`ipfs add -q afile` && + HASH=`ipfs add -q --pin=false afile` && ipfs pin add "$HASH" ' test_expect_success "recursively pin rm afile (default action)" ' - ipfs pin rm "$HASH" + ipfs pin rm "default/$HASH" ' test_expect_success "recursively pin afile" ' - ipfs pin add -r "$HASH" -' - -test_expect_success "pinning directly should fail now" ' - echo "Error: pin: $HASH already pinned recursively" >expected3 && - test_must_fail ipfs pin add -r=false "$HASH" 2>actual3 && - test_cmp expected3 actual3 -' - -test_expect_success "'ipfs pin rm -r=false ' should fail" ' - echo "Error: $HASH is not pinned directly" >expected4 && - test_must_fail ipfs pin rm -r=false "$HASH" 2>actual4 && - test_cmp expected4 actual4 -' - -test_expect_success "'ipfs pin rm -e -r=false ' should fail and explain why" ' - echo "Error: $HASH is pinned recursively" >expected11 && - test_must_fail ipfs pin rm -e -r=false "$HASH" 2>actual11 && - test_cmp expected11 actual11 + ipfs pin add "$HASH" ' -test_expect_success "remove recursive pin, add direct" ' - echo "unpinned $HASH" >expected5 && - ipfs pin rm -r "$HASH" >actual5 && - test_cmp expected5 actual5 && - ipfs pin add -r=false "$HASH" +test_expect_success "pinning directly should not fail either" ' + ipfs pin add -d "$HASH" ' -test_expect_success "remove direct pin" ' - echo "unpinned $HASH" >expected6 && - ipfs pin rm -r=false "$HASH" >actual6 && +test_expect_success "remove recursive pin" ' + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm "default/$HASH" >actual6 && test_cmp expected6 actual6 ' -test_expect_success "remove direct pin with pin rm -r" ' - echo "unpinned $HASH" >expected6 && - ipfs pin add -r=false "$HASH" - ipfs pin rm "$HASH" >actual6 && +test_expect_success "remove direct pin" ' + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm -d "default/$HASH" >actual6 && test_cmp expected6 actual6 ' @@ -146,37 +124,38 @@ test_expect_success "adding multiblock random file succeeds" ' ' test_expect_success "'ipfs pin ls --type=indirect' is correct" ' - ipfs refs "$MBLOCKHASH" >refsout && - ipfs refs -r "$HASH_WELCOME_DOCS" >>refsout && - sed -i"~" "s/\(.*\)/\1 indirect/g" refsout && - ipfs pin ls --type=indirect >indirectpins && + ipfs refs "$MBLOCKHASH" >mbrefsout && + ipfs refs -r "$HASH_WELCOME_DOCS" >assetsrefsout && + sed -i"~" "s|\(.*\)|\1 indirect added/$MBLOCKHASH|g" mbrefsout && + sed -i"~" "s/\(.*\)/\1 indirect assets/g" assetsrefsout && + ipfs pin ls -r --type=indirect >indirectpins && + cat assetsrefsout mbrefsout > refsout && test_sort_cmp refsout indirectpins ' test_expect_success "pin something directly" ' echo "ipfs is so awesome" >awesome && DIRECTPIN=`ipfs add -q awesome` && - echo "unpinned $DIRECTPIN" >expected9 && - ipfs pin rm -r "$DIRECTPIN" >actual9 && + echo "unpinned added/$DIRECTPIN" >expected9 && + ipfs pin rm "added/$DIRECTPIN" >actual9 && test_cmp expected9 actual9 && echo "pinned $DIRECTPIN directly" >expected10 && - ipfs pin add -r=false "$DIRECTPIN" >actual10 && + ipfs pin add -d "$DIRECTPIN" >actual10 && test_cmp expected10 actual10 ' test_expect_success "'ipfs pin ls --type=direct' is correct" ' - echo "$DIRECTPIN direct" >directpinexpected && - ipfs pin ls --type=direct >directpinout && + echo "$DIRECTPIN direct default/$DIRECTPIN" >directpinexpected && + ipfs pin ls -r --type=direct >directpinout && test_sort_cmp directpinexpected directpinout ' test_expect_success "'ipfs pin ls --type=recursive' is correct" ' - echo "$MBLOCKHASH" >rp_expected && - echo "$HASH_WELCOME_DOCS" >>rp_expected && - echo "$EMPTY_DIR" >>rp_expected && - sed -i"~" "s/\(.*\)/\1 recursive/g" rp_expected && - ipfs pin ls --type=recursive >rp_actual && + echo "$MBLOCKHASH added/$MBLOCKHASH" >rp_expected && + echo "$HASH_WELCOME_DOCS assets" >>rp_expected && + sed -i"~" -E "s/^([^ ]+)/\1 recursive/g" rp_expected && + ipfs pin ls -r --type=recursive >rp_actual && test_sort_cmp rp_expected rp_actual ' @@ -185,7 +164,7 @@ test_expect_success "'ipfs pin ls --type=all --quiet' is correct" ' cat rp_actual >>allpins && cat indirectpins >>allpins && cut -f1 -d " " allpins | sort | uniq >> allpins_uniq_hashes && - ipfs pin ls --type=all --quiet >actual_allpins && + ipfs pin ls -r --type=all --quiet >actual_allpins && test_sort_cmp allpins_uniq_hashes actual_allpins ' diff --git a/test/sharness/t0081-repo-pinning.sh b/test/sharness/t0081-repo-pinning.sh index 54e64253c04..2610fbb4d35 100755 --- a/test/sharness/t0081-repo-pinning.sh +++ b/test/sharness/t0081-repo-pinning.sh @@ -15,7 +15,7 @@ test_pin_flag() { echo "test_pin_flag" "$@" - if ipfs pin ls --type="$ptype" "$object" >actual + if ipfs pin ls -r --type="$ptype" | grep "^$object" >actual then test "$expect" = "true" && return test_fsh cat actual @@ -141,7 +141,7 @@ test_expect_success "added dir was NOT pinned indirectly" ' ' test_expect_success "nothing is pinned directly" ' - ipfs pin ls --type=direct >actual4 && + ipfs pin ls -r --type=direct >actual4 && test_must_be_empty actual4 ' @@ -166,8 +166,8 @@ test_expect_success "objects are still there" ' ' test_expect_success "remove dir recursive pin succeeds" ' - echo "unpinned $HASH_DIR1" >expected5 && - ipfs pin rm -r "$HASH_DIR1" >actual5 && + echo "unpinned added/$HASH_DIR1" >expected5 && + ipfs pin rm "added/$HASH_DIR1" >actual5 && test_cmp expected5 actual5 ' @@ -178,16 +178,16 @@ test_expect_success "none are pinned any more" ' test_pin "$HASH_FILE3" && test_pin "$HASH_FILE2" && test_pin "$HASH_FILE1" && - test_pin "$HASH_DIR3" && - test_pin "$HASH_DIR4" && - test_pin "$HASH_DIR2" && + test_pin "$HASH_DIR3" && + test_pin "$HASH_DIR4" && + test_pin "$HASH_DIR2" && test_pin "$HASH_DIR1" ' test_expect_success "pin some directly and indirectly" ' - ipfs pin add -r=false "$HASH_DIR1" >actual7 && - ipfs pin add -r=true "$HASH_DIR2" >>actual7 && - ipfs pin add -r=false "$HASH_FILE1" >>actual7 && + ipfs pin add -d "$HASH_DIR1" >actual7 && + ipfs pin add "$HASH_DIR2" >>actual7 && + ipfs pin add -d "$HASH_FILE1" >>actual7 && echo "pinned $HASH_DIR1 directly" >expected7 && echo "pinned $HASH_DIR2 recursively" >>expected7 && echo "pinned $HASH_FILE1 directly" >>expected7 && @@ -195,10 +195,10 @@ test_expect_success "pin some directly and indirectly" ' ' test_expect_success "pin lists look good" ' - test_pin $HASH_DIR1 direct && - test_pin $HASH_DIR2 recursive && - test_pin $HASH_DIR3 && - test_pin $HASH_DIR4 indirect && + test_pin $HASH_DIR1 direct && + test_pin $HASH_DIR2 recursive && + test_pin $HASH_DIR3 && + test_pin $HASH_DIR4 indirect && test_pin $HASH_FILE1 indirect direct && test_pin $HASH_FILE2 indirect && test_pin $HASH_FILE3 && @@ -226,7 +226,7 @@ test_expect_success "some objects are still there" ' ipfs cat "$HASH_FILE1" >>actual8 && ipfs ls "$HASH_DIR4" >>actual8 && ipfs ls "$HASH_DIR2" >>actual8 && - ipfs object links "$HASH_DIR1" >>actual8 && + ipfs object links "$HASH_DIR1" >>actual8 && test_cmp expected8 actual8 ' @@ -238,8 +238,8 @@ test_expect_success "some are no longer there" ' ' test_expect_success "recursive pin fails without objects" ' - ipfs pin rm -r=false "$HASH_DIR1" && - test_must_fail ipfs pin add -r "$HASH_DIR1" 2>err_expected8 && + ipfs pin rm -d "default/$HASH_DIR1" && + test_must_fail ipfs pin add "$HASH_DIR1" 2>err_expected8 && grep "pin: merkledag: not found" err_expected8 || test_fsh cat err_expected8 ' diff --git a/test/sharness/t0085-pins.sh b/test/sharness/t0085-pins.sh index bdc285edb66..c351d11eb6e 100755 --- a/test/sharness/t0085-pins.sh +++ b/test/sharness/t0085-pins.sh @@ -47,25 +47,25 @@ test_pins() { test_expect_success "test pin ls hash" ' echo $HASH_B | test_must_fail grep /ipfs && # just to be sure - ipfs pin ls $HASH_B > ls_hash_out && - echo "$HASH_B recursive" > ls_hash_exp && + ipfs pin ls default/$HASH_B > ls_hash_out && + echo "$HASH_B recursive default/$HASH_B" > ls_hash_exp && test_cmp ls_hash_exp ls_hash_out ' test_expect_success "unpin those hashes" ' - cat hashes | ipfs pin rm + cat hashes | sed "s|\(.*\)|default/\1|" | ipfs pin rm ' test_expect_success "test pin update" ' - ipfs pin add "$HASH_A" && - ipfs pin ls > before_update && + ipfs pin add -P test/updated_pin "$HASH_A" && + ipfs pin ls -r > before_update && test_should_contain "$HASH_A" before_update && test_must_fail grep -q "$HASH_B" before_update && - ipfs pin update --unpin=true "$HASH_A" "$HASH_B" && - ipfs pin ls > after_update && + ipfs pin update test/updated_pin "$HASH_B" && + ipfs pin ls -r > after_update && test_must_fail grep -q "$HASH_A" after_update && test_should_contain "$HASH_B" after_update && - ipfs pin rm "$HASH_B" + ipfs pin rm test/updated_pin ' } @@ -93,11 +93,11 @@ test_pin_dag() { test_pin_dag_init $1 test_expect_success "'ipfs pin add --progress' file" ' - ipfs pin add --recursive=true $HASH + ipfs pin add $HASH ' test_expect_success "'ipfs pin rm' file" ' - ipfs pin rm $HASH + ipfs pin rm default/$HASH ' test_expect_success "remove part of the dag" ' @@ -106,7 +106,7 @@ test_pin_dag() { ' test_expect_success "pin file, should fail" ' - test_must_fail ipfs pin add --recursive=true $HASH 2> err && + test_must_fail ipfs pin add $HASH 2> err && cat err && grep -q "not found" err ' diff --git a/test/sharness/t0087-repo-robust-gc.sh b/test/sharness/t0087-repo-robust-gc.sh index 896fc6929b4..9057d788bf8 100755 --- a/test/sharness/t0087-repo-robust-gc.sh +++ b/test/sharness/t0087-repo-robust-gc.sh @@ -13,7 +13,7 @@ test_gc_robust_part1() { test_expect_success "add a 1MB file with --raw-leaves" ' random 1048576 56 > afile && - HASH1=`ipfs add --raw-leaves -q afile` + HASH1=`ipfs add --raw-leaves -q -P test/afile afile` ' HASH1FILE=.ipfs/blocks/L3/CIQNIPL4GP62ZMNNSLZ2G33Z3T5VAN3YHCJTGT5FG45XWH5FGZRXL3A.data @@ -56,7 +56,7 @@ test_gc_robust_part1() { ' test_expect_success "unpin the 1MB file" ' - ipfs pin rm $HASH1 + ipfs pin rm test/afile ' # make sure the permission problem is fixed on exit, otherwise cleanup @@ -91,7 +91,7 @@ test_gc_robust_part2() { test_expect_success "add 1MB file normally (i.e., without raw leaves)" ' random 1048576 56 > afile && - HASH2=`ipfs add -q afile` + HASH2=`ipfs add -q -P test/afile afile` ' LEAF1=QmSijovevteoY63Uj1uC5b8pkpDU5Jgyk2dYBqz3sMJUPc @@ -142,7 +142,7 @@ test_gc_robust_part2() { ' test_expect_success "unpin 1MB file" ' - ipfs pin rm $HASH2 + ipfs pin rm test/afile ' test_expect_success "'ipfs repo gc' should be fine now" ' diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 244fdcd0a57..f10df0aabca 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -92,7 +92,7 @@ test_sharding() { test_expect_success "can unpin a file from sharded directory $EXTRA" ' read -r _ HASH _ < pin_hash && - ipfs pin rm $HASH + ipfs pin rm default/$HASH ' test_expect_success "output object was really sharded and has correct hash $EXTRA" ' diff --git a/test/sharness/t0252-files-gc.sh b/test/sharness/t0252-files-gc.sh index eeb2552f80a..7dc67fbe920 100755 --- a/test/sharness/t0252-files-gc.sh +++ b/test/sharness/t0252-files-gc.sh @@ -29,12 +29,12 @@ test_expect_success "gc okay after adding incomplete node -- prep" ' ipfs files mkdir /adir && echo "file1" | ipfs files write --create /adir/file1 && echo "file2" | ipfs files write --create /adir/file2 && - ipfs pin add --recursive=false $ADIR_HASH && + ipfs pin add -d $ADIR_HASH && ipfs files rm -r /adir && ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir test_must_fail ipfs cat $FILE1_HASH && ipfs files cp /ipfs/$ADIR_HASH /adir && - ipfs pin rm -r=false $ADIR_HASH + ipfs pin rm -d default/$ADIR_HASH ' test_expect_success "gc okay after adding incomplete node" ' @@ -49,7 +49,7 @@ test_expect_success "add directory with direct pin" ' FILE_UNPINNED=$(ipfs add --pin=false -q -r mydir/hello.txt) && DIR_PINNED=$(ipfs add --pin=false -q -r mydir | tail -n1) && ipfs add --pin=false -r mydir && - ipfs pin add --recursive=false $DIR_PINNED && + ipfs pin add -d $DIR_PINNED && ipfs cat $FILE_UNPINNED ' diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 6fb3cf657d9..1f130fda49b 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -68,7 +68,7 @@ test_expect_success "ipfs cat error output the same" ' test_expect_success "'ipfs ls --resolve-type=false' admits missing block" ' ipfs ls "$SHARDED" | head -1 > first_file && read -r HASH _ NAME /dev/null ' @@ -126,7 +126,7 @@ test_expect_success "files can not be retrieved via the urlstore" ' ' test_expect_success "remove broken files" ' - ipfs pin rm $HASH2 && + ipfs pin rm default/$HASH2 && ipfs repo gc > /dev/null ' diff --git a/test/sharness/t0276-cidv0v1.sh b/test/sharness/t0276-cidv0v1.sh index b1e437a4308..0e0db0d8b43 100755 --- a/test/sharness/t0276-cidv0v1.sh +++ b/test/sharness/t0276-cidv0v1.sh @@ -59,7 +59,7 @@ test_expect_success "make sure the CIDv1 hash is not in the repo" ' ' test_expect_success "clean up" ' - ipfs pin rm $AHASHv0 && + ipfs pin rm added/$AHASHv0 && ipfs repo gc && ! ipfs refs local | grep -q $AHASHv0 ' diff --git a/test/sharness/t0600-issues-and-regressions-online.sh b/test/sharness/t0600-issues-and-regressions-online.sh index 7cc66019c1c..678fa05d818 100755 --- a/test/sharness/t0600-issues-and-regressions-online.sh +++ b/test/sharness/t0600-issues-and-regressions-online.sh @@ -35,14 +35,14 @@ test_expect_success "metrics work" ' test_expect_success "pin add api looks right - #3753" ' HASH=$(echo "foo" | ipfs add -q) && - curl "http://$API_ADDR/api/v0/pin/add/$HASH" > pinadd_out && + curl "http://$API_ADDR/api/v0/pin/add/$HASH?pinpath=pintest" > pinadd_out && echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinadd_exp && test_cmp pinadd_out pinadd_exp ' test_expect_success "pin add api looks right - #3753" ' - curl "http://$API_ADDR/api/v0/pin/rm/$HASH" > pinrm_out && - echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinrm_exp && + curl "http://$API_ADDR/api/v0/pin/rm/pintest" > pinrm_out && + echo "{\"Pins\":[\"pintest\"]}" > pinrm_exp && test_cmp pinrm_out pinrm_exp ' diff --git a/test/sharness/x0601-pin-fail-test.sh b/test/sharness/x0601-pin-fail-test.sh index ffab1062d07..301bc225f8d 100755 --- a/test/sharness/x0601-pin-fail-test.sh +++ b/test/sharness/x0601-pin-fail-test.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon test_expect_success "pre-test setup" ' printf "" > pins && - ipfs pin ls --type=recursive -q > rec_pins_before + ipfs pin ls -r --type=recursive -q > rec_pins_before ' @@ -26,7 +26,7 @@ do done test_expect_success "get pinset afterwards" ' - ipfs pin ls --type=recursive -q | sort > rec_pins_after && + ipfs pin ls -r --type=recursive -q | sort > rec_pins_after && cat pins rec_pins_before | sort | uniq > exp_pins_after && test_cmp rec_pins_after exp_pins_after '