Named pins, storing pins in Datastore

* Change pin structure to a tree stored in KV store * Adjust pin-related commands accordingly * Make pinning routines use meaningful default prefixes License: MIT Signed-off-by: Iaroslav Gridin <voker57@gmail.com>
ipfs · Sep 9, 2018 · 959a49a · 959a49a
1 parent 1c98f6f
commit 959a49a
Show file tree

Hide file tree

Showing 38 changed files with 644 additions and 1,220 deletions.
diff --git a/assets/assets.go b/assets/assets.go
@@ -80,13 +80,9 @@ func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
 		return nil, err
 	}
 
-	if err := nd.Pinning.Pin(nd.Context(), dir, true); err != nil {
+	if err := nd.Pinning.Pin(nd.Context(), "assets", dir, true); err != nil {
 		return nil, fmt.Errorf("assets: Pinning on init-docu failed: %s", err)
 	}
 
-	if err := nd.Pinning.Flush(); err != nil {
-		return nil, fmt.Errorf("assets: Pinning flush failed: %s", err)
-	}
-
 	return dir.Cid(), nil
 }
diff --git a/core/builder.go b/core/builder.go
@@ -259,15 +259,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 	n.Blocks = bserv.New(n.Blockstore, n.Exchange)
 	n.DAG = dag.NewDAGService(n.Blocks)
 
-	internalDag := dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore)))
-	n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG, internalDag)
-	if err != nil {
-		// TODO: we should move towards only running 'NewPinner' explicitly on
-		// node init instead of implicitly here as a result of the pinner keys
-		// not being found in the datastore.
-		// this is kinda sketchy and could cause data loss
-		n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG, internalDag)
-	}
+	n.Pinning = pin.NewPinner(n.DAG, n.Repo.Datastore())
 	n.Resolver = resolver.NewBasicResolver(n.DAG)
 
 	if cfg.Online {

diff --git a/core/commands/add.go b/core/commands/add.go
@@ -41,6 +41,7 @@ const (
 	onlyHashOptionName    = "only-hash"
 	chunkerOptionName     = "chunker"
 	pinOptionName         = "pin"
+	pinPathOptionName     = "pinpath"
 	rawLeavesOptionName   = "raw-leaves"
 	noCopyOptionName      = "nocopy"
 	fstoreCacheOptionName = "fscache"
@@ -119,6 +120,7 @@ You can now check what blocks have been created by:
 		cmdkit.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
 		cmdkit.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").WithDefault("size-262144"),
 		cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
+		cmdkit.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"),
 		cmdkit.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
 		cmdkit.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
@@ -174,6 +176,7 @@ You can now check what blocks have been created by:
 		silent, _ := req.Options[silentOptionName].(bool)
 		chunker, _ := req.Options[chunkerOptionName].(string)
 		dopin, _ := req.Options[pinOptionName].(bool)
+		pinPath, _ := req.Options[pinPathOptionName].(string)
 		rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
 		nocopy, _ := req.Options[noCopyOptionName].(bool)
 		fscache, _ := req.Options[fstoreCacheOptionName].(bool)
@@ -278,6 +281,7 @@ You can now check what blocks have been created by:
 		}
 
 		fileAdder.Out = outChan
+		fileAdder.PinPath = pinPath
 		fileAdder.Chunker = chunker
 		fileAdder.Progress = progress
 		fileAdder.Hidden = hidden

diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go
@@ -10,7 +10,6 @@ import (
 	cmds "github.com/ipfs/go-ipfs/commands"
 	e "github.com/ipfs/go-ipfs/core/commands/e"
 	coredag "github.com/ipfs/go-ipfs/core/coredag"
-	pin "github.com/ipfs/go-ipfs/pin"
 	path "gx/ipfs/QmNgXoHgXU1HzNb2HEZmRww9fDKE9NfDsvQwWLHiKHpvKM/go-path"
 
 	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
@@ -62,7 +61,7 @@ into an object of the specified format.
 	Options: []cmdkit.Option{
 		cmdkit.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"),
 		cmdkit.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"),
-		cmdkit.BoolOption("pin", "Pin this object when adding."),
+		cmdkit.StringOption("pin", "Pin this object when adding.").WithDefault(""),
 		cmdkit.StringOption("hash", "Hash function to use").WithDefault(""),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
@@ -75,7 +74,7 @@ into an object of the specified format.
 		ienc, _, _ := req.Option("input-enc").String()
 		format, _, _ := req.Option("format").String()
 		hash, _, err := req.Option("hash").String()
-		dopin, _, err := req.Option("pin").Bool()
+		pinpath, _, err := req.Option("pin").String()
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
@@ -140,15 +139,12 @@ into an object of the specified format.
 				return err
 			}
 
-			if dopin {
-				defer n.Blockstore.PinLock().Unlock()
+			if pinpath != "" {
 
-				cids.ForEach(func(c *cid.Cid) error {
-					n.Pinning.PinWithMode(c, pin.Recursive)
-					return nil
+				err := cids.ForEach(func(c *cid.Cid) error {
+					return n.Pinning.AddPin(pinpath, c, true)
 				})
 
-				err := n.Pinning.Flush()
 				if err != nil {
 					return err
 				}

diff --git a/core/commands/object/object.go b/core/commands/object/object.go
@@ -160,7 +160,7 @@ multihash.
 			}
 		}
 
-		out := Object{
+		out := &Object{
 			Hash:  rp.Cid().String(),
 			Links: outLinks,
 		}
@@ -436,6 +436,7 @@ And then run:
 		cmdkit.StringOption("inputenc", "Encoding type of input data. One of: {\"protobuf\", \"json\"}.").WithDefault("json"),
 		cmdkit.StringOption("datafieldenc", "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"),
 		cmdkit.BoolOption("pin", "Pin this object when adding."),
+		cmdkit.StringOption("pinpath", "Pin under this path").WithDefault("added/"),
 		cmdkit.BoolOption("quiet", "q", "Write minimal output."),
 	},
 	Run: func(req oldcmds.Request, res oldcmds.Response) {
@@ -463,7 +464,13 @@ And then run:
 			return
 		}
 
-		dopin, _, err := req.Option("pin").Bool()
+		pinpath, _, err := req.Option("pinpath").String()
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		pin, _, err := req.Option("pin").Bool()
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
@@ -472,7 +479,8 @@ And then run:
 		p, err := api.Object().Put(req.Context(), input,
 			options.Object.DataType(datafieldenc),
 			options.Object.InputEnc(inputenc),
-			options.Object.Pin(dopin))
+			options.Object.Pin(pin),
+			options.Object.PinPath(pinpath))
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return