From 9fc54f6313ef519ce84126ff0421eae60cd6d769 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 30 Sep 2020 10:57:50 -0400 Subject: [PATCH 01/29] start restructuring for move to datastore pinner --- dspinner/pin.go | 516 +++++++++++++++++++++++++ dspinner/pin_test.go | 3 + go.mod | 1 + ipldpinner/pin.go | 535 ++++++++++++++++++++++++++ ipldpinner/pin_test.go | 3 + set.go => ipldpinner/set.go | 2 +- set_test.go => ipldpinner/set_test.go | 2 +- pin.go | 497 +----------------------- 8 files changed, 1065 insertions(+), 494 deletions(-) create mode 100644 dspinner/pin.go create mode 100644 dspinner/pin_test.go create mode 100644 ipldpinner/pin.go create mode 100644 ipldpinner/pin_test.go rename set.go => ipldpinner/set.go (99%) rename set_test.go => ipldpinner/set_test.go (99%) diff --git a/dspinner/pin.go b/dspinner/pin.go new file mode 100644 index 0000000..34f351d --- /dev/null +++ b/dspinner/pin.go @@ -0,0 +1,516 @@ +// Package pin implements structures and methods to keep track of +// which objects a user wants to keep stored locally. +package dspinner + +import ( + "context" + "fmt" + "sync" + "time" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" + mdag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" + + + cbor "github.com/polydawn/refmt/cbor" + ipfspinner "github.com/ipfs/go-ipfs-pinner" +) + +var log = logging.Logger("pin") + +var pinDatastoreKey = ds.NewKey("/.pins") + +var linkDirect, linkRecursive, linkInternal string +func init() { + directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) + if !ok { + panic("could not find Direct pin enum") + } + linkDirect = directStr + + recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive) + if !ok { + panic("could not find Recursive pin enum") + } + linkRecursive = recursiveStr + + internalStr, ok := ipfspinner.ModeToString(ipfspinner.Internal) + if !ok { + panic("could not find Internal pin enum") + } + linkInternal = internalStr +} + +// pinner implements the Pinner interface +type pinner struct { + lock sync.RWMutex + recursePin *cid.Set + directPin *cid.Set + + dserv ipld.DAGService + dstore ds.Datastore +} + +type pin struct { + depth int + version int + codec int + metadata map[string]interface{} +} + +var _ ipfspinner.Pinner = (*pinner)(nil) + +type syncDAGService interface { + ipld.DAGService + Sync() error +} + +// NewPinner creates a new pinner using the given datastore as a backend +func NewPinner(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { + return &pinner{ + dserv: serv, + dstore: dstore, + } +} + +// Pin the given node, optionally recursive +func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { + err := p.dserv.Add(ctx, node) + if err != nil { + return err + } + + c := node.Cid() + + p.lock.Lock() + defer p.lock.Unlock() + + if recurse { + if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Recursive) { + return nil + } + + p.lock.Unlock() + // temporary unlock to fetch the entire graph + err := mdag.FetchGraph(ctx, c, p.dserv) + p.lock.Lock() + if err != nil { + return err + } + + if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Recursive) { + return nil + } + + if p.directPin.Has(c) { + p.directPin.Remove(c) + } + + p.recursePin.Add(c) + } else { + if p.recursePin.Has(c) { + return fmt.Errorf("%s already pinned recursively", c.String()) + } + + p.directPin.Add(c) + } + return nil +} + +// ErrNotPinned is returned when trying to unpin items which are not pinned. +var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + +// Unpin a given key +func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + p.lock.Lock() + defer p.lock.Unlock() + if p.recursePin.Has(c) { + if !recursive { + return fmt.Errorf("%s is pinned recursively", c) + } + p.recursePin.Remove(c) + return nil + } + if p.directPin.Has(c) { + p.directPin.Remove(c) + return nil + } + return ErrNotPinned +} + +// IsPinned returns whether or not the given key is pinned +// and an explanation of why its pinned +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, ipfspinner.Any) +} + +// IsPinnedWithType returns whether or not the given cid is pinned with the +// given pin type, as well as returning the type of pin its pinned with. +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, mode) +} + +func (p *pinner) isPinnedWithTypeBool(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) bool { + _, found, _ := p.isPinnedWithType(ctx, c, mode) + return found +} + +func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + switch mode { + case ipfspinner.Any, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal: + default: + err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal, ipfspinner.Any) + return "", false, err + } + if (mode == ipfspinner.Recursive || mode == ipfspinner.Any) && p.recursePin.Has(c) { + return linkRecursive, true, nil + } + if mode == ipfspinner.Recursive { + return "", false, nil + } + + if (mode == ipfspinner.Direct || mode == ipfspinner.Any) && p.directPin.Has(c) { + return linkDirect, true, nil + } + if mode == ipfspinner.Direct { + return "", false, nil + } + + if mode == ipfspinner.Internal { + return "", false, nil + } + + // Default is Indirect + visitedSet := cid.NewSet() + for _, rc := range p.recursePin.Keys() { + has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if err != nil { + return "", false, err + } + if has { + return rc.String(), true, nil + } + } + return "", false, nil +} + +// CheckIfPinned Checks if a set of keys are pinned, more efficient than +// calling IsPinned for each key, returns the pinned status of cid(s) +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { + p.lock.RLock() + defer p.lock.RUnlock() + pinned := make([]ipfspinner.Pinned, 0, len(cids)) + toCheck := cid.NewSet() + + // First check for non-Indirect pins directly + for _, c := range cids { + if p.recursePin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) + } else if p.directPin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else { + toCheck.Add(c) + } + } + + // Now walk all recursive pins to check for indirect pins + var checkChildren func(cid.Cid, cid.Cid) error + checkChildren = func(rk, parentKey cid.Cid) error { + links, err := ipld.GetLinks(ctx, p.dserv, parentKey) + if err != nil { + return err + } + for _, lnk := range links { + c := lnk.Cid + + if toCheck.Has(c) { + pinned = append(pinned, + ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk}) + toCheck.Remove(c) + } + + err := checkChildren(rk, c) + if err != nil { + return err + } + + if toCheck.Len() == 0 { + return nil + } + } + return nil + } + + for _, rk := range p.recursePin.Keys() { + err := checkChildren(rk, rk) + if err != nil { + return nil, err + } + if toCheck.Len() == 0 { + break + } + } + + // Anything left in toCheck is not pinned + for _, k := range toCheck.Keys() { + pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned}) + } + + return pinned, nil +} + +// RemovePinWithMode is for manually editing the pin structure. +// Use with care! If used improperly, garbage collection may not +// be successful. +func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Direct: + p.directPin.Remove(c) + case ipfspinner.Recursive: + p.recursePin.Remove(c) + default: + // programmer error, panic OK + panic("unrecognized pin type") + } +} + +func cidSetWithValues(cids []cid.Cid) *cid.Set { + out := cid.NewSet() + for _, c := range cids { + out.Add(c) + } + return out +} + +// LoadPinner loads a pinner and its keysets from the given datastore +func LoadPinner(d ds.Datastore, dserv ipld.DAGService) (*pinner, error) { + p := new(pinner) + + rootKey, err := d.Get(pinDatastoreKey) + if err != nil { + return nil, fmt.Errorf("cannot load pin state: %v", err) + } + rootCid, err := cid.Cast(rootKey) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) + defer cancel() + + root, err := internal.Get(ctx, rootCid) + if err != nil { + return nil, fmt.Errorf("cannot find pinning root object: %v", err) + } + + rootpb, ok := root.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + + internalset := cid.NewSet() + internalset.Add(rootCid) + recordInternal := internalset.Add + + { // load recursive set + recurseKeys, err := ipldpinner.loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load recursive pins: %v", err) + } + p.recursePin = cidSetWithValues(recurseKeys) + } + + { // load direct set + directKeys, err := ipldpinner.loadSet(ctx, internal, rootpb, linkDirect, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load direct pins: %v", err) + } + p.directPin = cidSetWithValues(directKeys) + } + + p.internalPin = internalset + + // assign services + p.dserv = dserv + p.dstore = d + p.internal = internal + + return p, nil +} + +// DirectKeys returns a slice containing the directly pinned keys +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.directPin.Keys(), nil +} + +// RecursiveKeys returns a slice containing the recursively pinned keys +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.recursePin.Keys(), nil +} + +// Update updates a recursive pin from one cid to another +// this is more efficient than simply pinning the new one and unpinning the +// old one +func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { + if from == to { + // Nothing to do. Don't remove this check or we'll end up + // _removing_ the pin. + // + // See #6648 + return nil + } + + p.lock.Lock() + defer p.lock.Unlock() + + if !p.recursePin.Has(from) { + return fmt.Errorf("'from' cid was not recursively pinned already") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + + p.recursePin.Add(to) + if unpin { + p.recursePin.Remove(from) + } + return nil +} + +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() + + internalset := cid.NewSet() + recordInternal := internalset.Add + + root := &mdag.ProtoNode{} + { + n, err := ipldpinner.storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkDirect, n); err != nil { + return err + } + } + + { + n, err := ipldpinner.storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkRecursive, n); err != nil { + return err + } + } + + // add the empty node, its referenced by the pin sets but never created + err := p.internal.Add(ctx, new(mdag.ProtoNode)) + if err != nil { + return err + } + + err = p.internal.Add(ctx, root) + if err != nil { + return err + } + + k := root.Cid() + + internalset.Add(k) + + if syncDServ, ok := p.dserv.(syncDAGService); ok { + if err := syncDServ.Sync(); err != nil { + return fmt.Errorf("cannot sync pinned data: %v", err) + } + } + + if syncInternal, ok := p.internal.(syncDAGService); ok { + if err := syncInternal.Sync(); err != nil { + return fmt.Errorf("cannot sync pinning data: %v", err) + } + } + + if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { + return fmt.Errorf("cannot store pin state: %v", err) + } + if err := p.dstore.Sync(pinDatastoreKey); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } + p.internalPin = internalset + return nil +} + +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + p.lock.Lock() + defer p.lock.Unlock() + var out []cid.Cid + out = append(out, p.internalPin.Keys()...) + return out, nil +} + +// PinWithMode allows the user to have fine grained control over pin +// counts +func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Recursive: + p.recursePin.Add(c) + case ipfspinner.Direct: + p.directPin.Add(c) + } +} + +// hasChild recursively looks for a Cid among the children of a root Cid. +// The visit function can be used to shortcut already-visited branches. +func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { + links, err := ipld.GetLinks(ctx, ng, root) + if err != nil { + return false, err + } + for _, lnk := range links { + c := lnk.Cid + if lnk.Cid.Equals(child) { + return true, nil + } + if visit(c) { + has, err := hasChild(ctx, ng, c, child, visit) + if err != nil { + return false, err + } + + if has { + return has, nil + } + } + } + return false, nil +} diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go new file mode 100644 index 0000000..b139b34 --- /dev/null +++ b/dspinner/pin_test.go @@ -0,0 +1,3 @@ +package dspinner + +// TODO Call root Pin tests + anything special here \ No newline at end of file diff --git a/go.mod b/go.mod index 48479ae..bd0869c 100644 --- a/go.mod +++ b/go.mod @@ -13,4 +13,5 @@ require ( github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.1 github.com/ipfs/go-merkledag v0.3.0 + github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 ) diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go new file mode 100644 index 0000000..d22e2eb --- /dev/null +++ b/ipldpinner/pin.go @@ -0,0 +1,535 @@ +// Package pin implements structures and methods to keep track of +// which objects a user wants to keep stored locally. +package ipldpinner + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" + mdag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" + + ipfspinner "github.com/ipfs/go-ipfs-pinner" +) + +var log = logging.Logger("pin") + +var pinDatastoreKey = ds.NewKey("/local/pins") + +var emptyKey cid.Cid + +var linkDirect, linkRecursive, linkInternal string +func init() { + e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + if err != nil { + log.Error("failed to decode empty key constant") + os.Exit(1) + } + emptyKey = e + + directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) + if !ok { + panic("could not find Direct pin enum") + } + linkDirect = directStr + + recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive) + if !ok { + panic("could not find Recursive pin enum") + } + linkRecursive = recursiveStr + + internalStr, ok := ipfspinner.ModeToString(ipfspinner.Internal) + if !ok { + panic("could not find Internal pin enum") + } + linkInternal = internalStr +} + +// pinner implements the Pinner interface +type pinner struct { + lock sync.RWMutex + recursePin *cid.Set + directPin *cid.Set + + // Track the keys used for storing the pinning state, so gc does + // not delete them. + internalPin *cid.Set + dserv ipld.DAGService + internal ipld.DAGService // dagservice used to store internal objects + dstore ds.Datastore +} + +var _ ipfspinner.Pinner = (*pinner)(nil) + +type syncDAGService interface { + ipld.DAGService + Sync() error +} + +// NewPinner creates a new pinner using the given datastore as a backend +func New(dstore ds.Datastore, serv, internal ipld.DAGService) *pinner { + + rcset := cid.NewSet() + dirset := cid.NewSet() + + return &pinner{ + recursePin: rcset, + directPin: dirset, + dserv: serv, + dstore: dstore, + internal: internal, + internalPin: cid.NewSet(), + } +} + +// Pin the given node, optionally recursive +func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { + err := p.dserv.Add(ctx, node) + if err != nil { + return err + } + + c := node.Cid() + + p.lock.Lock() + defer p.lock.Unlock() + + if recurse { + if p.recursePin.Has(c) { + return nil + } + + p.lock.Unlock() + // temporary unlock to fetch the entire graph + err := mdag.FetchGraph(ctx, c, p.dserv) + p.lock.Lock() + if err != nil { + return err + } + + if p.recursePin.Has(c) { + return nil + } + + if p.directPin.Has(c) { + p.directPin.Remove(c) + } + + p.recursePin.Add(c) + } else { + if p.recursePin.Has(c) { + return fmt.Errorf("%s already pinned recursively", c.String()) + } + + p.directPin.Add(c) + } + return nil +} + +// ErrNotPinned is returned when trying to unpin items which are not pinned. +var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + +// Unpin a given key +func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + p.lock.Lock() + defer p.lock.Unlock() + if p.recursePin.Has(c) { + if !recursive { + return fmt.Errorf("%s is pinned recursively", c) + } + p.recursePin.Remove(c) + return nil + } + if p.directPin.Has(c) { + p.directPin.Remove(c) + return nil + } + return ErrNotPinned +} + +func (p *pinner) isInternalPin(c cid.Cid) bool { + return p.internalPin.Has(c) +} + +// IsPinned returns whether or not the given key is pinned +// and an explanation of why its pinned +func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, ipfspinner.Any) +} + +// IsPinnedWithType returns whether or not the given cid is pinned with the +// given pin type, as well as returning the type of pin its pinned with. +func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isPinnedWithType(ctx, c, mode) +} + +// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. +// intended for use by other pinned methods that already take locks +func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + switch mode { + case ipfspinner.Any, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal: + default: + err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal, ipfspinner.Any) + return "", false, err + } + if (mode == ipfspinner.Recursive || mode == ipfspinner.Any) && p.recursePin.Has(c) { + return linkRecursive, true, nil + } + if mode == ipfspinner.Recursive { + return "", false, nil + } + + if (mode == ipfspinner.Direct || mode == ipfspinner.Any) && p.directPin.Has(c) { + return linkDirect, true, nil + } + if mode == ipfspinner.Direct { + return "", false, nil + } + + if (mode == ipfspinner.Internal || mode == ipfspinner.Any) && p.isInternalPin(c) { + return linkInternal, true, nil + } + if mode == ipfspinner.Internal { + return "", false, nil + } + + // Default is Indirect + visitedSet := cid.NewSet() + for _, rc := range p.recursePin.Keys() { + has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if err != nil { + return "", false, err + } + if has { + return rc.String(), true, nil + } + } + return "", false, nil +} + +// CheckIfPinned Checks if a set of keys are pinned, more efficient than +// calling IsPinned for each key, returns the pinned status of cid(s) +func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { + p.lock.RLock() + defer p.lock.RUnlock() + pinned := make([]ipfspinner.Pinned, 0, len(cids)) + toCheck := cid.NewSet() + + // First check for non-Indirect pins directly + for _, c := range cids { + if p.recursePin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) + } else if p.directPin.Has(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else if p.isInternalPin(c) { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Internal}) + } else { + toCheck.Add(c) + } + } + + // Now walk all recursive pins to check for indirect pins + var checkChildren func(cid.Cid, cid.Cid) error + checkChildren = func(rk, parentKey cid.Cid) error { + links, err := ipld.GetLinks(ctx, p.dserv, parentKey) + if err != nil { + return err + } + for _, lnk := range links { + c := lnk.Cid + + if toCheck.Has(c) { + pinned = append(pinned, + ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk}) + toCheck.Remove(c) + } + + err := checkChildren(rk, c) + if err != nil { + return err + } + + if toCheck.Len() == 0 { + return nil + } + } + return nil + } + + for _, rk := range p.recursePin.Keys() { + err := checkChildren(rk, rk) + if err != nil { + return nil, err + } + if toCheck.Len() == 0 { + break + } + } + + // Anything left in toCheck is not pinned + for _, k := range toCheck.Keys() { + pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned}) + } + + return pinned, nil +} + +// RemovePinWithMode is for manually editing the pin structure. +// Use with care! If used improperly, garbage collection may not +// be successful. +func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Direct: + p.directPin.Remove(c) + case ipfspinner.Recursive: + p.recursePin.Remove(c) + default: + // programmer error, panic OK + panic("unrecognized pin type") + } +} + +func cidSetWithValues(cids []cid.Cid) *cid.Set { + out := cid.NewSet() + for _, c := range cids { + out.Add(c) + } + return out +} + +// LoadPinner loads a pinner and its keysets from the given datastore +func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { + p := new(pinner) + + rootKey, err := d.Get(pinDatastoreKey) + if err != nil { + return nil, fmt.Errorf("cannot load pin state: %v", err) + } + rootCid, err := cid.Cast(rootKey) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) + defer cancel() + + root, err := internal.Get(ctx, rootCid) + if err != nil { + return nil, fmt.Errorf("cannot find pinning root object: %v", err) + } + + rootpb, ok := root.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + + internalset := cid.NewSet() + internalset.Add(rootCid) + recordInternal := internalset.Add + + { // load recursive set + recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load recursive pins: %v", err) + } + p.recursePin = cidSetWithValues(recurseKeys) + } + + { // load direct set + directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load direct pins: %v", err) + } + p.directPin = cidSetWithValues(directKeys) + } + + p.internalPin = internalset + + // assign services + p.dserv = dserv + p.dstore = d + p.internal = internal + + return p, nil +} + +// DirectKeys returns a slice containing the directly pinned keys +func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.directPin.Keys(), nil +} + +// RecursiveKeys returns a slice containing the recursively pinned keys +func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.recursePin.Keys(), nil +} + +// Update updates a recursive pin from one cid to another +// this is more efficient than simply pinning the new one and unpinning the +// old one +func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { + if from == to { + // Nothing to do. Don't remove this check or we'll end up + // _removing_ the pin. + // + // See #6648 + return nil + } + + p.lock.Lock() + defer p.lock.Unlock() + + if !p.recursePin.Has(from) { + return fmt.Errorf("'from' cid was not recursively pinned already") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + + p.recursePin.Add(to) + if unpin { + p.recursePin.Remove(from) + } + return nil +} + +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() + + internalset := cid.NewSet() + recordInternal := internalset.Add + + root := &mdag.ProtoNode{} + { + n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkDirect, n); err != nil { + return err + } + } + + { + n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) + if err != nil { + return err + } + if err := root.AddNodeLink(linkRecursive, n); err != nil { + return err + } + } + + // add the empty node, its referenced by the pin sets but never created + err := p.internal.Add(ctx, new(mdag.ProtoNode)) + if err != nil { + return err + } + + err = p.internal.Add(ctx, root) + if err != nil { + return err + } + + k := root.Cid() + + internalset.Add(k) + + if syncDServ, ok := p.dserv.(syncDAGService); ok { + if err := syncDServ.Sync(); err != nil { + return fmt.Errorf("cannot sync pinned data: %v", err) + } + } + + if syncInternal, ok := p.internal.(syncDAGService); ok { + if err := syncInternal.Sync(); err != nil { + return fmt.Errorf("cannot sync pinning data: %v", err) + } + } + + if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { + return fmt.Errorf("cannot store pin state: %v", err) + } + if err := p.dstore.Sync(pinDatastoreKey); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } + p.internalPin = internalset + return nil +} + +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + p.lock.Lock() + defer p.lock.Unlock() + var out []cid.Cid + out = append(out, p.internalPin.Keys()...) + return out, nil +} + +// PinWithMode allows the user to have fine grained control over pin +// counts +func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { + p.lock.Lock() + defer p.lock.Unlock() + switch mode { + case ipfspinner.Recursive: + p.recursePin.Add(c) + case ipfspinner.Direct: + p.directPin.Add(c) + } +} + +// hasChild recursively looks for a Cid among the children of a root Cid. +// The visit function can be used to shortcut already-visited branches. +func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { + links, err := ipld.GetLinks(ctx, ng, root) + if err != nil { + return false, err + } + for _, lnk := range links { + c := lnk.Cid + if lnk.Cid.Equals(child) { + return true, nil + } + if visit(c) { + has, err := hasChild(ctx, ng, c, child, visit) + if err != nil { + return false, err + } + + if has { + return has, nil + } + } + } + return false, nil +} diff --git a/ipldpinner/pin_test.go b/ipldpinner/pin_test.go new file mode 100644 index 0000000..27ecccd --- /dev/null +++ b/ipldpinner/pin_test.go @@ -0,0 +1,3 @@ +package ipldpinner + +// TODO Call root Pin tests + anything special here \ No newline at end of file diff --git a/set.go b/ipldpinner/set.go similarity index 99% rename from set.go rename to ipldpinner/set.go index ca43797..fdb460d 100644 --- a/set.go +++ b/ipldpinner/set.go @@ -1,4 +1,4 @@ -package pin +package ipldpinner import ( "bytes" diff --git a/set_test.go b/ipldpinner/set_test.go similarity index 99% rename from set_test.go rename to ipldpinner/set_test.go index 61a3118..0f32e6b 100644 --- a/set_test.go +++ b/ipldpinner/set_test.go @@ -1,4 +1,4 @@ -package pin +package ipldpinner import ( "context" diff --git a/pin.go b/pin.go index aa74c51..98717ce 100644 --- a/pin.go +++ b/pin.go @@ -5,33 +5,16 @@ package pin import ( "context" "fmt" - "os" - "sync" - "time" + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-ipfs-pinner/dspinner" cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" - mdag "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-merkledag/dagutils" ) var log = logging.Logger("pin") -var pinDatastoreKey = ds.NewKey("/local/pins") - -var emptyKey cid.Cid - -func init() { - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) - } - emptyKey = e -} - const ( linkRecursive = "recursive" linkDirect = "direct" @@ -178,481 +161,11 @@ func (p Pinned) String() string { } } -// pinner implements the Pinner interface -type pinner struct { - lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - - // Track the keys used for storing the pinning state, so gc does - // not delete them. - internalPin *cid.Set - dserv ipld.DAGService - internal ipld.DAGService // dagservice used to store internal objects - dstore ds.Datastore -} - -type syncDAGService interface { - ipld.DAGService - Sync() error -} - -// NewPinner creates a new pinner using the given datastore as a backend func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() - - return &pinner{ - recursePin: rcset, - directPin: dirset, - dserv: serv, - dstore: dstore, - internal: internal, - internalPin: cid.NewSet(), - } -} - -// Pin the given node, optionally recursive -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { - err := p.dserv.Add(ctx, node) - if err != nil { - return err - } - - c := node.Cid() - - p.lock.Lock() - defer p.lock.Unlock() - - if recurse { - if p.recursePin.Has(c) { - return nil - } - - p.lock.Unlock() - // temporary unlock to fetch the entire graph - err := mdag.FetchGraph(ctx, c, p.dserv) - p.lock.Lock() - if err != nil { - return err - } - - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - - p.recursePin.Add(c) - } else { - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) - } - - p.directPin.Add(c) - } - return nil -} - -// ErrNotPinned is returned when trying to unpin items which are not pinned. -var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") - -// Unpin a given key -func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { - p.lock.Lock() - defer p.lock.Unlock() - if p.recursePin.Has(c) { - if !recursive { - return fmt.Errorf("%s is pinned recursively", c) - } - p.recursePin.Remove(c) - return nil - } - if p.directPin.Has(c) { - p.directPin.Remove(c) - return nil - } - return ErrNotPinned -} - -func (p *pinner) isInternalPin(c cid.Cid) bool { - return p.internalPin.Has(c) -} - -// IsPinned returns whether or not the given key is pinned -// and an explanation of why its pinned -func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, Any) -} - -// IsPinnedWithType returns whether or not the given cid is pinned with the -// given pin type, as well as returning the type of pin its pinned with. -func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(ctx, c, mode) -} - -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. -// intended for use by other pinned methods that already take locks -func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) { - switch mode { - case Any, Direct, Indirect, Recursive, Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, Direct, Indirect, Recursive, Internal, Any) - return "", false, err - } - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == Recursive { - return "", false, nil - } - - if (mode == Direct || mode == Any) && p.directPin.Has(c) { - return linkDirect, true, nil - } - if mode == Direct { - return "", false, nil - } - - if (mode == Internal || mode == Any) && p.isInternalPin(c) { - return linkInternal, true, nil - } - if mode == Internal { - return "", false, nil - } - - // Default is Indirect - visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err - } - if has { - return rc.String(), true, nil - } - } - return "", false, nil -} - -// CheckIfPinned Checks if a set of keys are pinned, more efficient than -// calling IsPinned for each key, returns the pinned status of cid(s) -func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() - pinned := make([]Pinned, 0, len(cids)) - toCheck := cid.NewSet() - - // First check for non-Indirect pins directly - for _, c := range cids { - if p.recursePin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Direct}) - } else if p.isInternalPin(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Internal}) - } else { - toCheck.Add(c) - } - } - - // Now walk all recursive pins to check for indirect pins - var checkChildren func(cid.Cid, cid.Cid) error - checkChildren = func(rk, parentKey cid.Cid) error { - links, err := ipld.GetLinks(ctx, p.dserv, parentKey) - if err != nil { - return err - } - for _, lnk := range links { - c := lnk.Cid - - if toCheck.Has(c) { - pinned = append(pinned, - Pinned{Key: c, Mode: Indirect, Via: rk}) - toCheck.Remove(c) - } - - err := checkChildren(rk, c) - if err != nil { - return err - } - - if toCheck.Len() == 0 { - return nil - } - } - return nil - } - - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) - if err != nil { - return nil, err - } - if toCheck.Len() == 0 { - break - } - } - - // Anything left in toCheck is not pinned - for _, k := range toCheck.Keys() { - pinned = append(pinned, Pinned{Key: k, Mode: NotPinned}) - } - - return pinned, nil -} - -// RemovePinWithMode is for manually editing the pin structure. -// Use with care! If used improperly, garbage collection may not -// be successful. -func (p *pinner) RemovePinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Direct: - p.directPin.Remove(c) - case Recursive: - p.recursePin.Remove(c) - default: - // programmer error, panic OK - panic("unrecognized pin type") - } -} - -func cidSetWithValues(cids []cid.Cid) *cid.Set { - out := cid.NewSet() - for _, c := range cids { - out.Add(c) - } - return out + return dspinner.NewPinner(dstore, serv) } // LoadPinner loads a pinner and its keysets from the given datastore func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - p := new(pinner) - - rootKey, err := d.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) - defer cancel() - - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add - - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - p.recursePin = cidSetWithValues(recurseKeys) - } - - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) - } - - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil -} - -// DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.directPin.Keys(), nil -} - -// RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.recursePin.Keys(), nil -} - -// Update updates a recursive pin from one cid to another -// this is more efficient than simply pinning the new one and unpinning the -// old one -func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { - if from == to { - // Nothing to do. Don't remove this check or we'll end up - // _removing_ the pin. - // - // See #6648 - return nil - } - - p.lock.Lock() - defer p.lock.Unlock() - - if !p.recursePin.Has(from) { - return fmt.Errorf("'from' cid was not recursively pinned already") - } - - // Temporarily unlock while we fetch the differences. - p.lock.Unlock() - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) - p.lock.Lock() - - if err != nil { - return err - } - - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush(ctx context.Context) error { - p.lock.Lock() - defer p.lock.Unlock() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } - } - - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) - if err != nil { - return err - } - - err = p.internal.Add(ctx, root) - if err != nil { - return err - } - - k := root.Cid() - - internalset.Add(k) - - if syncDServ, ok := p.dserv.(syncDAGService); ok { - if err := syncDServ.Sync(); err != nil { - return fmt.Errorf("cannot sync pinned data: %v", err) - } - } - - if syncInternal, ok := p.internal.(syncDAGService); ok { - if err := syncInternal.Sync(); err != nil { - return fmt.Errorf("cannot sync pinning data: %v", err) - } - } - - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - if err := p.dstore.Sync(pinDatastoreKey); err != nil { - return fmt.Errorf("cannot sync pin state: %v", err) - } - p.internalPin = internalset - return nil -} - -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { - p.lock.Lock() - defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out, nil -} - -// PinWithMode allows the user to have fine grained control over pin -// counts -func (p *pinner) PinWithMode(c cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Recursive: - p.recursePin.Add(c) - case Direct: - p.directPin.Add(c) - } -} - -// hasChild recursively looks for a Cid among the children of a root Cid. -// The visit function can be used to shortcut already-visited branches. -func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) { - links, err := ipld.GetLinks(ctx, ng, root) - if err != nil { - return false, err - } - for _, lnk := range links { - c := lnk.Cid - if lnk.Cid.Equals(child) { - return true, nil - } - if visit(c) { - has, err := hasChild(ctx, ng, c, child, visit) - if err != nil { - return false, err - } - - if has { - return has, nil - } - } - } - return false, nil -} + return dspinner.LoadPinner(d, dserv) +} \ No newline at end of file From be03825ca03962aa817249a2ba2d4489972eaaa8 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 26 Oct 2020 14:39:34 -0700 Subject: [PATCH 02/29] Store pins in datastore instead of in mdag Pins are stored in the datastore as separate key-value items. This allows pins to be saved (flushed) without havint to hash the entire pin set into a hierarchical dag on each flush. This also means there are no longer any need for internal pins to pin the blocks used to store the pin dag. Secondary indexes are also supported, allowing for pins to be searched for using keys othen than the primary key. This supports multiple pins for the same CID as well as search by different pin attributes, when those features become available. --- .gitignore | 8 + dsindex/indexer.go | 173 +++++++++++ dsindex/indexer_test.go | 173 +++++++++++ dspinner/pin.go | 621 ++++++++++++++++++++++++++++------------ dspinner/pin_test.go | 415 ++++++++++++++++++++++++++- ipldpinner/pin.go | 77 +++-- ipldpinner/pin_test.go | 416 ++++++++++++++++++++++++++- ipldpinner/set.go | 14 +- pin.go | 11 - pin_test.go | 416 --------------------------- 10 files changed, 1676 insertions(+), 648 deletions(-) create mode 100644 .gitignore create mode 100644 dsindex/indexer.go create mode 100644 dsindex/indexer_test.go delete mode 100644 pin_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c34288 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*~ +*.log + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool +*.out diff --git a/dsindex/indexer.go b/dsindex/indexer.go new file mode 100644 index 0000000..85da5e5 --- /dev/null +++ b/dsindex/indexer.go @@ -0,0 +1,173 @@ +// Package dsindex provides secondary indexing functionality for a datastore. +package dsindex + +import ( + "path" + + ds "github.com/ipfs/go-datastore" + query "github.com/ipfs/go-datastore/query" +) + +// Indexer maintains a secondary index. Each value of the secondary index maps +// to one more primary keys. +type Indexer interface { + // Add adds the a to the an index + Add(index, id string) error + + // Has determines if a key is in an index + Has(index, id string) (bool, error) + + // Delete deletes the specified key from the index. If the key is not in + // the datastore, this method returns no error. + Delete(index, id string) error + + // DeleteAll deletes all keys in the given index. If a key is not in the + // datastore, this method returns no error. + DeleteAll(index string) (count int, err error) + + // Search returns all keys for the given index + Search(index string) (ids []string, err error) + + // All returns a map of key to secondary index value for all indexed keys + All() (map[string]string, error) + + // Synchronize the indexes in this Indexer to match those of the given + // Indexer. The indexPath prefix is not synchronized, only the index/key + // portion of the indexes. + SyncTo(reference Indexer) (changed bool, err error) +} + +// indexer is a simple implementation of Indexer. This implementation relies +// on the underlying data store supporting efficent querying by prefix. +// +// TODO: Consider adding caching +type indexer struct { + dstore ds.Datastore + indexPath string +} + +// New creates a new datastore index. All indexes are stored prefixed with the +// specified index path. +func New(dstore ds.Datastore, indexPath string) Indexer { + return &indexer{ + dstore: dstore, + indexPath: indexPath, + } +} + +func (x *indexer) Add(index, id string) error { + key := ds.NewKey(path.Join(x.indexPath, index, id)) + return x.dstore.Put(key, []byte{}) +} + +func (x *indexer) Has(index, id string) (bool, error) { + key := ds.NewKey(path.Join(x.indexPath, index, id)) + return x.dstore.Has(key) +} + +func (x *indexer) Delete(index, id string) error { + key := ds.NewKey(path.Join(x.indexPath, index, id)) + return x.dstore.Delete(key) +} + +func (x *indexer) DeleteAll(index string) (int, error) { + ids, err := x.Search(index) + if err != nil { + return 0, err + } + for i := range ids { + err = x.Delete(index, ids[i]) + if err != nil { + return 0, err + } + } + return len(ids), nil +} + +func (x *indexer) Search(index string) ([]string, error) { + ents, err := x.queryPrefix(path.Join(x.indexPath, index)) + if err != nil { + return nil, err + } + if len(ents) == 0 { + return nil, nil + } + + ids := make([]string, len(ents)) + for i := range ents { + ids[i] = path.Base(ents[i].Key) + } + return ids, nil +} + +func (x *indexer) All() (map[string]string, error) { + ents, err := x.queryPrefix(x.indexPath) + if err != nil { + return nil, err + } + if len(ents) == 0 { + return nil, nil + } + + indexes := make(map[string]string, len(ents)) + for i := range ents { + fullPath := ents[i].Key + indexes[path.Base(fullPath)] = path.Base(path.Dir(fullPath)) + } + + return indexes, nil +} + +func (x *indexer) SyncTo(ref Indexer) (changed bool, err error) { + var curAll, refAll map[string]string + + refAll, err = ref.All() + if err != nil { + return + } + + curAll, err = x.All() + if err != nil { + return + } + + for k, v := range refAll { + cv, ok := curAll[k] + if ok && cv == v { + // same in both, so delete from both + delete(curAll, k) + delete(refAll, k) + } + } + + // What remains in curAll are indexes that no longer exist + for k, v := range curAll { + err = x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, v, k))) + if err != nil { + return + } + } + + // What remains in refAll are indexes that need to be added + for k, v := range refAll { + err = x.dstore.Put(ds.NewKey(path.Join(x.indexPath, v, k)), nil) + if err != nil { + return + } + } + + changed = len(refAll) != 0 || len(curAll) != 0 + return +} + +func (x *indexer) queryPrefix(prefix string) ([]query.Entry, error) { + q := query.Query{ + Prefix: prefix, + KeysOnly: true, + } + results, err := x.dstore.Query(q) + if err != nil { + return nil, err + } + return results.Rest() +} diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go new file mode 100644 index 0000000..93546c4 --- /dev/null +++ b/dsindex/indexer_test.go @@ -0,0 +1,173 @@ +package dsindex + +import ( + "testing" + + ds "github.com/ipfs/go-datastore" +) + +func createIndexer() Indexer { + dstore := ds.NewMapDatastore() + nameIndex := New(dstore, "/data/nameindex") + + nameIndex.Add("alice", "a1") + nameIndex.Add("bob", "b1") + nameIndex.Add("bob", "b2") + nameIndex.Add("cathy", "c1") + + return nameIndex +} + +func TestAdd(t *testing.T) { + nameIndex := createIndexer() + err := nameIndex.Add("someone", "s1") + if err != nil { + t.Fatal(err) + } + err = nameIndex.Add("someone", "s1") + if err != nil { + t.Fatal(err) + } + err = nameIndex.Add("", "s1") + if err != nil { + t.Fatal(err) + } +} + +func TestHas(t *testing.T) { + nameIndex := createIndexer() + + ok, err := nameIndex.Has("bob", "b1") + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing index") + } + + ok, err = nameIndex.Has("bob", "b3") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should not have index") + } +} + +func TestSearch(t *testing.T) { + nameIndex := createIndexer() + + ids, err := nameIndex.Search("bob") + if err != nil { + t.Fatal(err) + } + if len(ids) != 2 { + t.Fatal("wrong number of ids - expected 2 got", ids) + } + for _, id := range ids { + if id != "b1" && id != "b2" { + t.Fatal("wrong value in id set") + } + } + if ids[0] == ids[1] { + t.Fatal("dublicate id") + } + + ids, err = nameIndex.Search("cathy") + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0] != "c1" { + t.Fatal("wrong ids") + } + + ids, err = nameIndex.Search("amit") + if err != nil { + t.Fatal(err) + } + if len(ids) != 0 { + t.Fatal("unexpected ids returned") + } +} + +func TestDelete(t *testing.T) { + nameIndex := createIndexer() + + err := nameIndex.Delete("bob", "b3") + if err != nil { + t.Fatal(err) + } + + err = nameIndex.Delete("alice", "a1") + if err != nil { + t.Fatal(err) + } + + ok, err := nameIndex.Has("alice", "a1") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("index should have been deleted") + } + + count, err := nameIndex.DeleteAll("bob") + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("wrong deleted count") + } + ok, _ = nameIndex.Has("bob", "b1") + if ok { + t.Fatal("index not deleted") + } +} + +func TestAll(t *testing.T) { + nameIndex := createIndexer() + + all, err := nameIndex.All() + if err != nil { + t.Fatal(err) + } + if len(all) != 4 { + t.Fatal("wrong number of indexes") + } + for k, v := range all { + if ok, _ := nameIndex.Has(v, k); !ok { + t.Fatal("indexes from all do not match what indexer has") + } + } +} + +func TestSyndTo(t *testing.T) { + nameIndex := createIndexer() + + dstore := ds.NewMapDatastore() + refIndex := New(dstore, "/ref") + refIndex.Add("alice", "a1") + refIndex.Add("cathy", "zz") + refIndex.Add("dennis", "d1") + + changed, err := nameIndex.SyncTo(refIndex) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Error("change was not indicated") + } + + refAll, _ := refIndex.All() + syncAll, _ := nameIndex.All() + + if len(syncAll) != len(refAll) { + t.Fatal("wrong number of indexes after sync") + } + for k, v := range refAll { + vSync, ok := syncAll[k] + if !ok || v != vSync { + t.Fatal("index", v, "-->", k, "was not synced") + } + } +} diff --git a/dspinner/pin.go b/dspinner/pin.go index 34f351d..0973ca2 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -3,28 +3,42 @@ package dspinner import ( + "bytes" "context" "fmt" + "path" "sync" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ipfspinner "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dsindex" + "github.com/ipfs/go-ipfs-pinner/ipldpinner" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" mdag "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-merkledag/dagutils" + "github.com/polydawn/refmt/cbor" +) +const ( + loadTimeout = 5 * time.Second - cbor "github.com/polydawn/refmt/cbor" - ipfspinner "github.com/ipfs/go-ipfs-pinner" + pinKeyPath = "/.pins/pin" + indexKeyPath = "/.pins/index" ) -var log = logging.Logger("pin") +var ( + // ErrNotPinned is returned when trying to unpin items that are not pinned. + ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + + log = logging.Logger("pin") -var pinDatastoreKey = ds.NewKey("/.pins") + linkDirect, linkRecursive string + pinCidIndexPath, pinNameIndexPath string +) -var linkDirect, linkRecursive, linkInternal string func init() { directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct) if !ok { @@ -38,42 +52,61 @@ func init() { } linkRecursive = recursiveStr - internalStr, ok := ipfspinner.ModeToString(ipfspinner.Internal) - if !ok { - panic("could not find Internal pin enum") - } - linkInternal = internalStr + pinCidIndexPath = path.Join(indexKeyPath, "cidindex") + pinNameIndexPath = path.Join(indexKeyPath, "nameindex") } // pinner implements the Pinner interface type pinner struct { - lock sync.RWMutex + lock sync.RWMutex + recursePin *cid.Set directPin *cid.Set - dserv ipld.DAGService - dstore ds.Datastore + dserv ipld.DAGService + dstore ds.Datastore + + cidIndex dsindex.Indexer } +var _ ipfspinner.Pinner = (*pinner)(nil) + type pin struct { - depth int - version int - codec int - metadata map[string]interface{} + id string + cid cid.Cid + metadata map[string]interface{} + mode ipfspinner.Mode + name string +} + +func (p *pin) codec() uint64 { return p.cid.Type() } +func (p *pin) version() uint64 { return p.cid.Version() } +func (p *pin) dsKey() ds.Key { + return ds.NewKey(path.Join(pinKeyPath, p.id)) } -var _ ipfspinner.Pinner = (*pinner)(nil) +func newPin(c cid.Cid, mode ipfspinner.Mode, name string) *pin { + return &pin{ + id: ds.RandomKey().String(), + cid: c, + name: name, + mode: mode, + } +} type syncDAGService interface { ipld.DAGService Sync() error } -// NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { +// New creates a new pinner using the given datastore as a backend +func New(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { return &pinner{ - dserv: serv, - dstore: dstore, + cidIndex: dsindex.New(dstore, pinCidIndexPath), + dserv: serv, + dstore: dstore, + directPin: cid.NewSet(), + recursePin: cid.NewSet(), } } @@ -106,40 +139,117 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return nil } - if p.directPin.Has(c) { - p.directPin.Remove(c) + // TODO: remove this to support multiple pins per CID + if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Direct) { + ok, _ := p.removePinsForCid(c, ipfspinner.Direct) + if !ok { + // Fix cache + p.directPin.Remove(c) + } } - p.recursePin.Add(c) + err = p.addPin(c, ipfspinner.Recursive, "") + if err != nil { + return err + } } else { if p.recursePin.Has(c) { return fmt.Errorf("%s already pinned recursively", c.String()) } + err = p.addPin(c, ipfspinner.Direct, "") + if err != nil { + return err + } + } + return nil +} + +func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { + // Create new pin and store in datastore + pp := newPin(c, mode, name) + + // Serialize pin + pinData, err := encodePin(pp) + if err != nil { + return fmt.Errorf("could not encode pin: %v", err) + } + + // Store CID index + err = p.cidIndex.Add(c.String(), pp.id) + if err != nil { + return fmt.Errorf("could not add pin cid index: %v", err) + } + + // Store the pin + err = p.dstore.Put(pp.dsKey(), pinData) + if err != nil { + p.cidIndex.Delete(c.String(), pp.id) + return err + } + + // Update cache + switch mode { + case ipfspinner.Recursive: + p.recursePin.Add(c) + case ipfspinner.Direct: p.directPin.Add(c) } + return nil } -// ErrNotPinned is returned when trying to unpin items which are not pinned. -var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") +func (p *pinner) removePin(pin *pin) error { + // Remove pin from datastore + err := p.dstore.Delete(pin.dsKey()) + if err != nil { + return err + } + // Remove cid index from datastore + err = p.cidIndex.Delete(pin.cid.String(), pin.id) + if err != nil { + return err + } + + // Update cache + switch pin.mode { + case ipfspinner.Recursive: + p.recursePin.Remove(pin.cid) + case ipfspinner.Direct: + p.directPin.Remove(pin.cid) + } + + return nil +} // Unpin a given key func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { p.lock.Lock() defer p.lock.Unlock() + + // TODO: use Ls() to lookup pins when new pinning API available + /* + matchSpec := map[string][]string { + "cid": []string{c.String} + } + matches := p.Ls(matchSpec) + */ if p.recursePin.Has(c) { if !recursive { return fmt.Errorf("%s is pinned recursively", c) } - p.recursePin.Remove(c) - return nil + } else if !p.directPin.Has(c) { + return ErrNotPinned } - if p.directPin.Has(c) { - p.directPin.Remove(c) - return nil + + ok, err := p.removePinsForCid(c, ipfspinner.Any) + if err != nil { + return err } - return ErrNotPinned + if !ok { + log.Error("found CID index with missing pin") + } + return nil } // IsPinned returns whether or not the given key is pinned @@ -165,28 +275,32 @@ func (p *pinner) isPinnedWithTypeBool(ctx context.Context, c cid.Cid, mode ipfsp func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { switch mode { - case ipfspinner.Any, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, ipfspinner.Internal, ipfspinner.Any) - return "", false, err - } - if (mode == ipfspinner.Recursive || mode == ipfspinner.Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == ipfspinner.Recursive { + case ipfspinner.Recursive: + if p.recursePin.Has(c) { + return linkRecursive, true, nil + } return "", false, nil - } - - if (mode == ipfspinner.Direct || mode == ipfspinner.Any) && p.directPin.Has(c) { - return linkDirect, true, nil - } - if mode == ipfspinner.Direct { + case ipfspinner.Direct: + if p.directPin.Has(c) { + return linkDirect, true, nil + } return "", false, nil - } - - if mode == ipfspinner.Internal { + case ipfspinner.Internal: return "", false, nil + case ipfspinner.Indirect: + case ipfspinner.Any: + if p.recursePin.Has(c) { + return linkRecursive, true, nil + } + if p.directPin.Has(c) { + return linkDirect, true, nil + } + default: + err := fmt.Errorf( + "invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", + mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive, + ipfspinner.Internal, ipfspinner.Any) + return "", false, err } // Default is Indirect @@ -205,6 +319,8 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne // CheckIfPinned Checks if a set of keys are pinned, more efficient than // calling IsPinned for each key, returns the pinned status of cid(s) +// +// TODO: If a CID is pinned by multiple pins, should they all be reported? func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { p.lock.RLock() defer p.lock.RUnlock() @@ -274,79 +390,159 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { p.lock.Lock() defer p.lock.Unlock() + + // Check cache to see if CID is pinned switch mode { case ipfspinner.Direct: - p.directPin.Remove(c) + if !p.directPin.Has(c) { + return + } case ipfspinner.Recursive: - p.recursePin.Remove(c) + if !p.recursePin.Has(c) { + return + } default: // programmer error, panic OK panic("unrecognized pin type") } + + p.removePinsForCid(c, mode) } -func cidSetWithValues(cids []cid.Cid) *cid.Set { - out := cid.NewSet() - for _, c := range cids { - out.Add(c) +// removePinsForCid removes all pins for a cid that have the specified mode. +func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) { + // Search for pins by CID + ids, err := p.cidIndex.Search(c.String()) + if err != nil { + return false, err } - return out + + var removed bool + + // Remove the pin with the requested mode + for _, pid := range ids { + var pp *pin + pp, err = p.loadPin(pid) + if err != nil { + if err == ds.ErrNotFound { + continue + } + return false, err + } + if mode == ipfspinner.Any || pp.mode == mode { + err = p.removePin(pp) + if err != nil { + return false, err + } + removed = true + } + } + return removed, nil } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv ipld.DAGService) (*pinner, error) { - p := new(pinner) +// loadPin loads a single pin from the datastore. +func (p *pinner) loadPin(pid string) (*pin, error) { + pinData, err := p.dstore.Get(ds.NewKey(path.Join(pinKeyPath, pid))) + if err != nil { + return nil, err + } + return decodePin(pid, pinData) +} - rootKey, err := d.Get(pinDatastoreKey) +// loadAllPins loads all pins from the datastore. +func (p *pinner) loadAllPins() ([]*pin, error) { + q := query.Query{ + Prefix: pinKeyPath, + } + results, err := p.dstore.Query(q) if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) + return nil, err } - rootCid, err := cid.Cast(rootKey) + ents, err := results.Rest() if err != nil { return nil, err } + if len(ents) == 0 { + return nil, nil + } + + pins := make([]*pin, len(ents)) + for i := range ents { + var p *pin + p, err := decodePin(path.Base(ents[i].Key), ents[i].Value) + if err != nil { + return nil, err + } + pins[i] = p + } + return pins, nil +} - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) +// LoadPinner loads a pinner and its keysets from the given datastore +func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, error) { + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) defer cancel() - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) + if internal != nil { + ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) + switch err { + case ds.ErrNotFound: + // No more dag storage; load from datastore + case nil: + // Need to convert from dag storage + return convertIPLDToDSPinner(ctx, ipldPinner, dstore, dserv) + default: + return nil, err + } } - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf + p := New(dstore, dserv).(*pinner) + err := p.rebuildIndexes(ctx) + if err != nil { + return nil, fmt.Errorf("cannot rebuild indexes: %v", err) } - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add + return p, nil +} - { // load recursive set - recurseKeys, err := ipldpinner.loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - p.recursePin = cidSetWithValues(recurseKeys) +// rebuildIndexes uses the stored pins to rebuild secondary indexes. This +// resolves any discrepancy between secondary indexes and pins that could +// result from a program termination between saving the two. +func (p *pinner) rebuildIndexes(ctx context.Context) error { + pins, err := p.loadAllPins() + if err != nil { + return fmt.Errorf("cannot load pins: %v", err) } - { // load direct set - directKeys, err := ipldpinner.loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) + p.directPin = cid.NewSet() + p.recursePin = cid.NewSet() + + // Build temporary in-memory CID index from pins + dstoreMem := ds.NewMapDatastore() + tmpCidIndex := dsindex.New(dstoreMem, pinCidIndexPath) + for _, pp := range pins { + tmpCidIndex.Add(pp.cid.String(), pp.id) + + // Build up cache + if pp.mode == ipfspinner.Recursive { + p.recursePin.Add(pp.cid) + } else if pp.mode == ipfspinner.Direct { + p.directPin.Add(pp.cid) } - p.directPin = cidSetWithValues(directKeys) } - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal + // Sync the CID index to what was build from pins. This fixes any invalid + // indexes, which could happen if ipfs was terminated between writing pin + // and writing secondary index. + changed, err := p.cidIndex.SyncTo(tmpCidIndex) + if err != nil { + return fmt.Errorf("cannot sync indexes: %v", err) + } + if changed { + log.Error("invalid indexes detected - rebuilt") + } - return p, nil + return nil } // DirectKeys returns a slice containing the directly pinned keys @@ -365,15 +561,18 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { return p.recursePin.Keys(), nil } -// Update updates a recursive pin from one cid to another -// this is more efficient than simply pinning the new one and unpinning the -// old one +// InternalPins returns all cids kept pinned for the internal state of the +// pinner +func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { + return nil, nil +} + +// Update updates a recursive pin from one cid to another. This is equivalent +// to pinning the new one and unpinning the old one. +// +// TODO: This will not work when multiple pins are supported func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { if from == to { - // Nothing to do. Don't remove this check or we'll end up - // _removing_ the pin. - // - // See #6648 return nil } @@ -384,65 +583,30 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return fmt.Errorf("'from' cid was not recursively pinned already") } - // Temporarily unlock while we fetch the differences. - p.lock.Unlock() - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) - p.lock.Lock() - + err := p.addPin(to, ipfspinner.Recursive, "") if err != nil { return err } - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush(ctx context.Context) error { - p.lock.Lock() - defer p.lock.Unlock() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := ipldpinner.storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := ipldpinner.storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } + if !unpin { + return nil } - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) + ok, err := p.removePinsForCid(from, ipfspinner.Recursive) if err != nil { return err } - - err = p.internal.Add(ctx, root) - if err != nil { - return err + if !ok { + log.Error("found CID index with missing pin") } - k := root.Cid() + return nil +} - internalset.Add(k) +// Flush encodes and writes pinner keysets to the datastore +func (p *pinner) Flush(ctx context.Context) error { + p.lock.Lock() + defer p.lock.Unlock() if syncDServ, ok := p.dserv.(syncDAGService); ok { if err := syncDServ.Sync(); err != nil { @@ -450,42 +614,31 @@ func (p *pinner) Flush(ctx context.Context) error { } } - if syncInternal, ok := p.internal.(syncDAGService); ok { - if err := syncInternal.Sync(); err != nil { - return fmt.Errorf("cannot sync pinning data: %v", err) - } - } + // TODO: is it necessary to keep a list of added pins to sync? + //if err := p.dstore.Sync(pinKey); err != nil { + // return fmt.Errorf("cannot sync pin state: %v", err) + //} - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - if err := p.dstore.Sync(pinDatastoreKey); err != nil { - return fmt.Errorf("cannot sync pin state: %v", err) - } - p.internalPin = internalset return nil } -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { - p.lock.Lock() - defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out, nil -} - // PinWithMode allows the user to have fine grained control over pin // counts func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { p.lock.Lock() defer p.lock.Unlock() - switch mode { - case ipfspinner.Recursive: - p.recursePin.Add(c) - case ipfspinner.Direct: - p.directPin.Add(c) + + ids, _ := p.cidIndex.Search(c.String()) + for i := range ids { + pp, _ := p.loadPin(ids[i]) + if pp != nil && pp.mode == mode { + return // already a pin for this CID with this mode + } + } + + err := p.addPin(c, mode, "") + if err != nil { + return } } @@ -514,3 +667,119 @@ func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.C } return false, nil } + +// convertIPLDToDSPinner converts pins stored in mdag based storage to pins +// stores in the datastore. After pins are stored in datastore, then root pin +// key is deleted to unlink the pin data in the mdag store. +func convertIPLDToDSPinner(ctx context.Context, ipldPinner ipfspinner.Pinner, dstore ds.Datastore, dserv ipld.DAGService) (*pinner, error) { + var err error + p := New(dstore, dserv).(*pinner) + + // Save pinned CIDs as new pins in datastore. + rCids, _ := ipldPinner.RecursiveKeys(ctx) + for i := range rCids { + err = p.addPin(rCids[i], ipfspinner.Recursive, "") + if err != nil { + return nil, err + } + } + dCids, _ := ipldPinner.DirectKeys(ctx) + for i := range rCids { + err = p.addPin(dCids[i], ipfspinner.Direct, "") + if err != nil { + return nil, err + } + } + + // Delete root mdag key from datastore to remove old pin storage. + pinDatastoreKey := ds.NewKey("/local/pins") + if err = dstore.Delete(pinDatastoreKey); err != nil { + return nil, fmt.Errorf("cannot delete old pin state: %v", err) + } + if err = dstore.Sync(pinDatastoreKey); err != nil { + return nil, fmt.Errorf("cannot sync old pin state: %v", err) + } + + return p, nil +} + +func encodePin(p *pin) ([]byte, error) { + var buf bytes.Buffer + encoder := cbor.NewMarshaller(&buf) + pinData := map[string]interface{}{ + "mode": p.mode, + "cid": p.cid.String(), + } + // Encode optional fields + if p.name != "" { + pinData["name"] = p.name + } + if len(p.metadata) != 0 { + pinData["metadata"] = p.metadata + } + + err := encoder.Marshal(pinData) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func decodePin(pid string, data []byte) (*pin, error) { + reader := bytes.NewReader(data) + decoder := cbor.NewUnmarshaller(cbor.DecodeOptions{}, reader) + + var pinData map[string]interface{} + err := decoder.Unmarshal(&pinData) + if err != nil { + return nil, fmt.Errorf("cannot decode pin: %v", err) + } + + cidData, ok := pinData["cid"] + if !ok { + return nil, fmt.Errorf("missing cid") + } + cidStr, ok := cidData.(string) + if !ok { + return nil, fmt.Errorf("invalid pin cid data") + } + c, err := cid.Decode(cidStr) + if err != nil { + return nil, fmt.Errorf("cannot decode pin cid: %v", err) + } + + modeData, ok := pinData["mode"] + if !ok { + return nil, fmt.Errorf("missing mode") + } + mode64, ok := modeData.(uint64) + if !ok { + return nil, fmt.Errorf("invalid pin mode data") + } + + p := &pin{ + id: pid, + mode: ipfspinner.Mode(mode64), + cid: c, + } + + // Decode optional data + + meta, ok := pinData["metadata"] + if ok && meta != nil { + p.metadata, ok = meta.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("cannot decode metadata") + } + } + + name, ok := pinData["name"] + if ok && name != nil { + p.name, ok = name.(string) + if !ok { + return nil, fmt.Errorf("invalid pin name data") + } + } + + return p, nil +} diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index b139b34..55c2ad7 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -1,3 +1,416 @@ package dspinner -// TODO Call root Pin tests + anything special here \ No newline at end of file +import ( + "context" + "io" + "testing" + "time" + + bs "github.com/ipfs/go-blockservice" + mdag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + ipfspin "github.com/ipfs/go-ipfs-pinner" + util "github.com/ipfs/go-ipfs-util" +) + +var rand = util.NewTimeSeededRand() + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func assertPinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatal(failmsg) + } +} + +func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatal(failmsg) + } +} + +func TestPinnerBasic(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p := New(dstore, dserv) + + a, ak := randNode() + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // Pin A{} + err = p.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ak, "Failed to find key") + + // create new node c, to be indirectly pinned through b + c, _ := randNode() + err = dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + + // Create new node b, to be parent to a and c + b, _ := randNode() + err = b.AddNodeLink("child", a) + if err != nil { + t.Fatal(err) + } + + err = b.AddNodeLink("otherchild", c) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + + // recursively pin B{A,C} + err = p.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ck, "child of recursively pinned node not found") + + assertPinned(t, p, bk, "Recursively pinned node not found..") + + d, _ := randNode() + _ = d.AddNodeLink("a", a) + _ = d.AddNodeLink("c", c) + + e, _ := randNode() + _ = d.AddNodeLink("e", e) + + // Must be in dagserv for unpin to work + err = dserv.Add(ctx, e) + if err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, d) + if err != nil { + t.Fatal(err) + } + + // Add D{A,C,E} + err = p.Pin(ctx, d, true) + if err != nil { + t.Fatal(err) + } + + dk := d.Cid() + assertPinned(t, p, dk, "pinned node not found.") + + // Test recursive unpin + err = p.Unpin(ctx, dk, true) + if err != nil { + t.Fatal(err) + } + + err = p.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + np, err := LoadPinner(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + + // Test directly pinned + assertPinned(t, np, ak, "Could not find pinned node!") + + // Test recursively pinned + assertPinned(t, np, bk, "could not find recursively pinned node") +} + +func TestIsPinnedLookup(t *testing.T) { + // We are going to test that lookups work in pins which share + // the same branches. For that we will construct this tree: + // + // A5->A4->A3->A2->A1->A0 + // / / + // B------- / + // \ / + // C--------------- + // + // We will ensure that IsPinned works for all objects both when they + // are pinned and once they have been unpinned. + aBranchLen := 6 + if aBranchLen < 3 { + t.Fatal("set aBranchLen to at least 3") + } + + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // TODO does pinner need to share datastore with blockservice? + p := New(dstore, dserv) + + aNodes := make([]*mdag.ProtoNode, aBranchLen) + aKeys := make([]cid.Cid, aBranchLen) + for i := 0; i < aBranchLen; i++ { + a, _ := randNode() + if i >= 1 { + err := a.AddNodeLink("child", aNodes[i-1]) + if err != nil { + t.Fatal(err) + } + } + + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + //t.Logf("a[%d] is %s", i, ak) + aNodes[i] = a + aKeys[i] = a.Cid() + } + + // Pin A5 recursively + if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + t.Fatal(err) + } + + // Create node B and add A3 as child + b, _ := randNode() + if err := b.AddNodeLink("mychild", aNodes[3]); err != nil { + t.Fatal(err) + } + + // Create C node + c, _ := randNode() + // Add A0 as child of C + if err := c.AddNodeLink("child", aNodes[0]); err != nil { + t.Fatal(err) + } + + // Add C + err := dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + //t.Logf("C is %s", ck) + + // Add C to B and Add B + if err := b.AddNodeLink("myotherchild", c); err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + //t.Logf("B is %s", bk) + + // Pin C recursively + + if err := p.Pin(ctx, c, true); err != nil { + t.Fatal(err) + } + + // Pin B recursively + + if err := p.Pin(ctx, b, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should be pinned") + assertPinned(t, p, aKeys[1], "A1 should be pinned") + assertPinned(t, p, ck, "C should be pinned") + assertPinned(t, p, bk, "B should be pinned") + + // Unpin A5 recursively + if err := p.Unpin(ctx, aKeys[5], true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") + assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") + + // Unpin B recursively + if err := p.Unpin(ctx, bk, true); err != nil { + t.Fatal(err) + } + assertUnpinned(t, p, bk, "B should be unpinned") + assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") + assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") +} + +func TestDuplicateSemantics(t *testing.T) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // TODO does pinner need to share datastore with blockservice? + p := New(dstore, dserv) + + a, _ := randNode() + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // pin is recursively + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + // pinning directly should fail + err = p.Pin(ctx, a, false) + if err == nil { + t.Fatal("expected direct pin to fail") + } + + // pinning recursively again should succeed + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestFlush(t *testing.T) { + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv) + _, k := randNode() + + p.PinWithMode(k, ipfspin.Recursive) + if err := p.Flush(context.Background()); err != nil { + t.Fatal(err) + } + assertPinned(t, p, k, "expected key to still be pinned") +} + +func TestPinRecursiveFail(t *testing.T) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p := New(dstore, dserv) + + a, _ := randNode() + b, _ := randNode() + err := a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + // NOTE: This isnt a time based test, we expect the pin to fail + mctx, cancel := context.WithTimeout(ctx, time.Millisecond) + defer cancel() + + err = p.Pin(mctx, a, true) + if err == nil { + t.Fatal("should have failed to pin here") + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // this one is time based... but shouldnt cause any issues + mctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + err = p.Pin(mctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestPinUpdate(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv) + n1, c1 := randNode() + n2, c2 := randNode() + + if err := dserv.Add(ctx, n1); err != nil { + t.Fatal(err) + } + if err := dserv.Add(ctx, n2); err != nil { + t.Fatal(err) + } + + if err := p.Pin(ctx, n1, true); err != nil { + t.Fatal(err) + } + + if err := p.Update(ctx, c1, c2, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned now") + assertUnpinned(t, p, c1, "c1 should no longer be pinned") + + if err := p.Update(ctx, c2, c1, false); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned still") + assertPinned(t, p, c1, "c1 should be pinned now") +} diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go index d22e2eb..6f28430 100644 --- a/ipldpinner/pin.go +++ b/ipldpinner/pin.go @@ -1,5 +1,6 @@ -// Package pin implements structures and methods to keep track of -// which objects a user wants to keep stored locally. +// Package ipldpinner implements structures and methods to keep track of +// which objects a user wants to keep stored locally. This implementation +// uses an IPLD DAG to determine indirect pins. package ipldpinner import ( @@ -19,6 +20,8 @@ import ( ipfspinner "github.com/ipfs/go-ipfs-pinner" ) +const loadTimeout = 5 * time.Second + var log = logging.Logger("pin") var pinDatastoreKey = ds.NewKey("/local/pins") @@ -26,6 +29,7 @@ var pinDatastoreKey = ds.NewKey("/local/pins") var emptyKey cid.Cid var linkDirect, linkRecursive, linkInternal string + func init() { e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") if err != nil { @@ -67,26 +71,22 @@ type pinner struct { dstore ds.Datastore } -var _ ipfspinner.Pinner = (*pinner)(nil) +var _ ipfspinner.Pinner = (*pinner)(nil) type syncDAGService interface { ipld.DAGService Sync() error } -// NewPinner creates a new pinner using the given datastore as a backend +// New creates a new pinner using the given datastore as a backend func New(dstore ds.Datastore, serv, internal ipld.DAGService) *pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() - return &pinner{ - recursePin: rcset, - directPin: dirset, + recursePin: cid.NewSet(), + directPin: cid.NewSet(), + internalPin: cid.NewSet(), dserv: serv, - dstore: dstore, internal: internal, - internalPin: cid.NewSet(), + dstore: dstore, } } @@ -313,11 +313,12 @@ func cidSetWithValues(cids []cid.Cid) *cid.Set { } // LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { - p := new(pinner) - - rootKey, err := d.Get(pinDatastoreKey) +func LoadPinner(dstore ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { + rootKey, err := dstore.Get(pinDatastoreKey) if err != nil { + if err == ds.ErrNotFound { + return nil, err + } return nil, fmt.Errorf("cannot load pin state: %v", err) } rootCid, err := cid.Cast(rootKey) @@ -325,7 +326,7 @@ func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error return nil, err } - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) defer cancel() root, err := internal.Get(ctx, rootCid) @@ -342,30 +343,28 @@ func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error internalset.Add(rootCid) recordInternal := internalset.Add - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - p.recursePin = cidSetWithValues(recurseKeys) + // load recursive set + recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load recursive pins: %v", err) } - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) + // load direct set + directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load direct pins: %v", err) } - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil + return &pinner{ + // assign pinsets + recursePin: cidSetWithValues(recurseKeys), + directPin: cidSetWithValues(directKeys), + internalPin: internalset, + // assign services + dserv: dserv, + dstore: dstore, + internal: internal, + }, nil } // DirectKeys returns a slice containing the directly pinned keys @@ -490,9 +489,7 @@ func (p *pinner) Flush(ctx context.Context) error { func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { p.lock.Lock() defer p.lock.Unlock() - var out []cid.Cid - out = append(out, p.internalPin.Keys()...) - return out, nil + return p.internalPin.Keys(), nil } // PinWithMode allows the user to have fine grained control over pin diff --git a/ipldpinner/pin_test.go b/ipldpinner/pin_test.go index 27ecccd..d2fbfb5 100644 --- a/ipldpinner/pin_test.go +++ b/ipldpinner/pin_test.go @@ -1,3 +1,417 @@ package ipldpinner -// TODO Call root Pin tests + anything special here \ No newline at end of file +import ( + "context" + "io" + "testing" + "time" + + bs "github.com/ipfs/go-blockservice" + mdag "github.com/ipfs/go-merkledag" + + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + pin "github.com/ipfs/go-ipfs-pinner" + util "github.com/ipfs/go-ipfs-util" +) + +var rand = util.NewTimeSeededRand() + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func assertPinned(t *testing.T, p pin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if !pinned { + t.Fatal(failmsg) + } +} + +func assertUnpinned(t *testing.T, p pin.Pinner, c cid.Cid, failmsg string) { + _, pinned, err := p.IsPinned(context.Background(), c) + if err != nil { + t.Fatal(err) + } + + if pinned { + t.Fatal(failmsg) + } +} + +func TestPinnerBasic(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // TODO does pinner need to share datastore with blockservice? + p := New(dstore, dserv, dserv) + + a, ak := randNode() + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // Pin A{} + err = p.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ak, "Failed to find key") + + // create new node c, to be indirectly pinned through b + c, _ := randNode() + err = dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + + // Create new node b, to be parent to a and c + b, _ := randNode() + err = b.AddNodeLink("child", a) + if err != nil { + t.Fatal(err) + } + + err = b.AddNodeLink("otherchild", c) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + + // recursively pin B{A,C} + err = p.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + assertPinned(t, p, ck, "child of recursively pinned node not found") + + assertPinned(t, p, bk, "Recursively pinned node not found..") + + d, _ := randNode() + _ = d.AddNodeLink("a", a) + _ = d.AddNodeLink("c", c) + + e, _ := randNode() + _ = d.AddNodeLink("e", e) + + // Must be in dagserv for unpin to work + err = dserv.Add(ctx, e) + if err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, d) + if err != nil { + t.Fatal(err) + } + + // Add D{A,C,E} + err = p.Pin(ctx, d, true) + if err != nil { + t.Fatal(err) + } + + dk := d.Cid() + assertPinned(t, p, dk, "pinned node not found.") + + // Test recursive unpin + err = p.Unpin(ctx, dk, true) + if err != nil { + t.Fatal(err) + } + + err = p.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + np, err := LoadPinner(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + + // Test directly pinned + assertPinned(t, np, ak, "Could not find pinned node!") + + // Test recursively pinned + assertPinned(t, np, bk, "could not find recursively pinned node") +} + +func TestIsPinnedLookup(t *testing.T) { + // We are going to test that lookups work in pins which share + // the same branches. For that we will construct this tree: + // + // A5->A4->A3->A2->A1->A0 + // / / + // B------- / + // \ / + // C--------------- + // + // We will ensure that IsPinned works for all objects both when they + // are pinned and once they have been unpinned. + aBranchLen := 6 + if aBranchLen < 3 { + t.Fatal("set aBranchLen to at least 3") + } + + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // TODO does pinner need to share datastore with blockservice? + p := New(dstore, dserv, dserv) + + aNodes := make([]*mdag.ProtoNode, aBranchLen) + aKeys := make([]cid.Cid, aBranchLen) + for i := 0; i < aBranchLen; i++ { + a, _ := randNode() + if i >= 1 { + err := a.AddNodeLink("child", aNodes[i-1]) + if err != nil { + t.Fatal(err) + } + } + + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + //t.Logf("a[%d] is %s", i, ak) + aNodes[i] = a + aKeys[i] = a.Cid() + } + + // Pin A5 recursively + if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + t.Fatal(err) + } + + // Create node B and add A3 as child + b, _ := randNode() + if err := b.AddNodeLink("mychild", aNodes[3]); err != nil { + t.Fatal(err) + } + + // Create C node + c, _ := randNode() + // Add A0 as child of C + if err := c.AddNodeLink("child", aNodes[0]); err != nil { + t.Fatal(err) + } + + // Add C + err := dserv.Add(ctx, c) + if err != nil { + t.Fatal(err) + } + ck := c.Cid() + //t.Logf("C is %s", ck) + + // Add C to B and Add B + if err := b.AddNodeLink("myotherchild", c); err != nil { + t.Fatal(err) + } + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + bk := b.Cid() + //t.Logf("B is %s", bk) + + // Pin C recursively + + if err := p.Pin(ctx, c, true); err != nil { + t.Fatal(err) + } + + // Pin B recursively + + if err := p.Pin(ctx, b, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should be pinned") + assertPinned(t, p, aKeys[1], "A1 should be pinned") + assertPinned(t, p, ck, "C should be pinned") + assertPinned(t, p, bk, "B should be pinned") + + // Unpin A5 recursively + if err := p.Unpin(ctx, aKeys[5], true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") + assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") + + // Unpin B recursively + if err := p.Unpin(ctx, bk, true); err != nil { + t.Fatal(err) + } + assertUnpinned(t, p, bk, "B should be unpinned") + assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") + assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") +} + +func TestDuplicateSemantics(t *testing.T) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + // TODO does pinner need to share datastore with blockservice? + p := New(dstore, dserv, dserv) + + a, _ := randNode() + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // pin is recursively + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + // pinning directly should fail + err = p.Pin(ctx, a, false) + if err == nil { + t.Fatal("expected direct pin to fail") + } + + // pinning recursively again should succeed + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestFlush(t *testing.T) { + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv, dserv) + _, k := randNode() + + p.PinWithMode(k, pin.Recursive) + if err := p.Flush(context.Background()); err != nil { + t.Fatal(err) + } + assertPinned(t, p, k, "expected key to still be pinned") +} + +func TestPinRecursiveFail(t *testing.T) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p := New(dstore, dserv, dserv) + + a, _ := randNode() + b, _ := randNode() + err := a.AddNodeLink("child", b) + if err != nil { + t.Fatal(err) + } + + // NOTE: This isnt a time based test, we expect the pin to fail + mctx, cancel := context.WithTimeout(ctx, time.Millisecond) + defer cancel() + + err = p.Pin(mctx, a, true) + if err == nil { + t.Fatal("should have failed to pin here") + } + + err = dserv.Add(ctx, b) + if err != nil { + t.Fatal(err) + } + + err = dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // this one is time based... but shouldnt cause any issues + mctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + err = p.Pin(mctx, a, true) + if err != nil { + t.Fatal(err) + } +} + +func TestPinUpdate(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv, dserv) + n1, c1 := randNode() + n2, c2 := randNode() + + if err := dserv.Add(ctx, n1); err != nil { + t.Fatal(err) + } + if err := dserv.Add(ctx, n2); err != nil { + t.Fatal(err) + } + + if err := p.Pin(ctx, n1, true); err != nil { + t.Fatal(err) + } + + if err := p.Update(ctx, c1, c2, true); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned now") + assertUnpinned(t, p, c1, "c1 should no longer be pinned") + + if err := p.Update(ctx, c2, c1, false); err != nil { + t.Fatal(err) + } + + assertPinned(t, p, c2, "c2 should be pinned still") + assertPinned(t, p, c1, "c1 should be pinned now") +} diff --git a/ipldpinner/set.go b/ipldpinner/set.go index fdb460d..2fb931f 100644 --- a/ipldpinner/set.go +++ b/ipldpinner/set.go @@ -55,9 +55,14 @@ func (s sortByHash) Swap(a, b int) { } func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - links := make([]*ipld.Link, 0, defaultFanout+maxItems) + // Each node wastes up to defaultFanout in empty links. + var leafLinks uint64 + if estimatedLen < maxItems { + leafLinks = estimatedLen + } + links := make([]*ipld.Link, defaultFanout, defaultFanout+leafLinks) for i := 0; i < defaultFanout; i++ { - links = append(links, &ipld.Link{Cid: emptyKey}) + links[i] = &ipld.Link{Cid: emptyKey} } // add emptyKey to our set of internal pinset objects @@ -97,7 +102,7 @@ func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, d sort.Stable(s) } - hashed := make([][]cid.Cid, defaultFanout) + var hashed [][]cid.Cid for { // This loop essentially enumerates every single item in the set // and maps them all into a set of buckets. Each bucket will be recursively @@ -116,6 +121,9 @@ func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, d if !ok { break } + if hashed == nil { + hashed = make([][]cid.Cid, defaultFanout) + } h := hash(depth, k) % defaultFanout hashed[h] = append(hashed[h], k) } diff --git a/pin.go b/pin.go index 98717ce..7e1d886 100644 --- a/pin.go +++ b/pin.go @@ -5,8 +5,6 @@ package pin import ( "context" "fmt" - ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-ipfs-pinner/dspinner" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -160,12 +158,3 @@ func (p Pinned) String() string { return fmt.Sprintf("pinned: %s", modeStr) } } - -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - return dspinner.NewPinner(dstore, serv) -} - -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - return dspinner.LoadPinner(d, dserv) -} \ No newline at end of file diff --git a/pin_test.go b/pin_test.go deleted file mode 100644 index e477ac0..0000000 --- a/pin_test.go +++ /dev/null @@ -1,416 +0,0 @@ -package pin - -import ( - "context" - "io" - "testing" - "time" - - bs "github.com/ipfs/go-blockservice" - mdag "github.com/ipfs/go-merkledag" - - cid "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" - offline "github.com/ipfs/go-ipfs-exchange-offline" - util "github.com/ipfs/go-ipfs-util" -) - -var rand = util.NewTimeSeededRand() - -func randNode() (*mdag.ProtoNode, cid.Cid) { - nd := new(mdag.ProtoNode) - nd.SetData(make([]byte, 32)) - _, err := io.ReadFull(rand, nd.Data()) - if err != nil { - panic(err) - } - k := nd.Cid() - return nd, k -} - -func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { - _, pinned, err := p.IsPinned(context.Background(), c) - if err != nil { - t.Fatal(err) - } - - if !pinned { - t.Fatal(failmsg) - } -} - -func assertUnpinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) { - _, pinned, err := p.IsPinned(context.Background(), c) - if err != nil { - t.Fatal(err) - } - - if pinned { - t.Fatal(failmsg) - } -} - -func TestPinnerBasic(t *testing.T) { - ctx := context.Background() - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - a, ak := randNode() - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // Pin A{} - err = p.Pin(ctx, a, false) - if err != nil { - t.Fatal(err) - } - - assertPinned(t, p, ak, "Failed to find key") - - // create new node c, to be indirectly pinned through b - c, _ := randNode() - err = dserv.Add(ctx, c) - if err != nil { - t.Fatal(err) - } - ck := c.Cid() - - // Create new node b, to be parent to a and c - b, _ := randNode() - err = b.AddNodeLink("child", a) - if err != nil { - t.Fatal(err) - } - - err = b.AddNodeLink("otherchild", c) - if err != nil { - t.Fatal(err) - } - - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - bk := b.Cid() - - // recursively pin B{A,C} - err = p.Pin(ctx, b, true) - if err != nil { - t.Fatal(err) - } - - assertPinned(t, p, ck, "child of recursively pinned node not found") - - assertPinned(t, p, bk, "Recursively pinned node not found..") - - d, _ := randNode() - _ = d.AddNodeLink("a", a) - _ = d.AddNodeLink("c", c) - - e, _ := randNode() - _ = d.AddNodeLink("e", e) - - // Must be in dagserv for unpin to work - err = dserv.Add(ctx, e) - if err != nil { - t.Fatal(err) - } - err = dserv.Add(ctx, d) - if err != nil { - t.Fatal(err) - } - - // Add D{A,C,E} - err = p.Pin(ctx, d, true) - if err != nil { - t.Fatal(err) - } - - dk := d.Cid() - assertPinned(t, p, dk, "pinned node not found.") - - // Test recursive unpin - err = p.Unpin(ctx, dk, true) - if err != nil { - t.Fatal(err) - } - - err = p.Flush(ctx) - if err != nil { - t.Fatal(err) - } - - np, err := LoadPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } - - // Test directly pinned - assertPinned(t, np, ak, "Could not find pinned node!") - - // Test recursively pinned - assertPinned(t, np, bk, "could not find recursively pinned node") -} - -func TestIsPinnedLookup(t *testing.T) { - // We are going to test that lookups work in pins which share - // the same branches. For that we will construct this tree: - // - // A5->A4->A3->A2->A1->A0 - // / / - // B------- / - // \ / - // C--------------- - // - // We will ensure that IsPinned works for all objects both when they - // are pinned and once they have been unpinned. - aBranchLen := 6 - if aBranchLen < 3 { - t.Fatal("set aBranchLen to at least 3") - } - - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - aNodes := make([]*mdag.ProtoNode, aBranchLen) - aKeys := make([]cid.Cid, aBranchLen) - for i := 0; i < aBranchLen; i++ { - a, _ := randNode() - if i >= 1 { - err := a.AddNodeLink("child", aNodes[i-1]) - if err != nil { - t.Fatal(err) - } - } - - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - //t.Logf("a[%d] is %s", i, ak) - aNodes[i] = a - aKeys[i] = a.Cid() - } - - // Pin A5 recursively - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { - t.Fatal(err) - } - - // Create node B and add A3 as child - b, _ := randNode() - if err := b.AddNodeLink("mychild", aNodes[3]); err != nil { - t.Fatal(err) - } - - // Create C node - c, _ := randNode() - // Add A0 as child of C - if err := c.AddNodeLink("child", aNodes[0]); err != nil { - t.Fatal(err) - } - - // Add C - err := dserv.Add(ctx, c) - if err != nil { - t.Fatal(err) - } - ck := c.Cid() - //t.Logf("C is %s", ck) - - // Add C to B and Add B - if err := b.AddNodeLink("myotherchild", c); err != nil { - t.Fatal(err) - } - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - bk := b.Cid() - //t.Logf("B is %s", bk) - - // Pin C recursively - - if err := p.Pin(ctx, c, true); err != nil { - t.Fatal(err) - } - - // Pin B recursively - - if err := p.Pin(ctx, b, true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, aKeys[0], "A0 should be pinned") - assertPinned(t, p, aKeys[1], "A1 should be pinned") - assertPinned(t, p, ck, "C should be pinned") - assertPinned(t, p, bk, "B should be pinned") - - // Unpin A5 recursively - if err := p.Unpin(ctx, aKeys[5], true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, aKeys[0], "A0 should still be pinned through B") - assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") - - // Unpin B recursively - if err := p.Unpin(ctx, bk, true); err != nil { - t.Fatal(err) - } - assertUnpinned(t, p, bk, "B should be unpinned") - assertUnpinned(t, p, aKeys[1], "A1 should be unpinned") - assertPinned(t, p, aKeys[0], "A0 should still be pinned through C") -} - -func TestDuplicateSemantics(t *testing.T) { - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - - // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) - - a, _ := randNode() - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // pin is recursively - err = p.Pin(ctx, a, true) - if err != nil { - t.Fatal(err) - } - - // pinning directly should fail - err = p.Pin(ctx, a, false) - if err == nil { - t.Fatal("expected direct pin to fail") - } - - // pinning recursively again should succeed - err = p.Pin(ctx, a, true) - if err != nil { - t.Fatal(err) - } -} - -func TestFlush(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - _, k := randNode() - - p.PinWithMode(k, Recursive) - if err := p.Flush(context.Background()); err != nil { - t.Fatal(err) - } - assertPinned(t, p, k, "expected key to still be pinned") -} - -func TestPinRecursiveFail(t *testing.T) { - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - dserv := mdag.NewDAGService(bserv) - - p := NewPinner(dstore, dserv, dserv) - - a, _ := randNode() - b, _ := randNode() - err := a.AddNodeLink("child", b) - if err != nil { - t.Fatal(err) - } - - // NOTE: This isnt a time based test, we expect the pin to fail - mctx, cancel := context.WithTimeout(ctx, time.Millisecond) - defer cancel() - - err = p.Pin(mctx, a, true) - if err == nil { - t.Fatal("should have failed to pin here") - } - - err = dserv.Add(ctx, b) - if err != nil { - t.Fatal(err) - } - - err = dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - - // this one is time based... but shouldnt cause any issues - mctx, cancel = context.WithTimeout(ctx, time.Second) - defer cancel() - err = p.Pin(mctx, a, true) - if err != nil { - t.Fatal(err) - } -} - -func TestPinUpdate(t *testing.T) { - ctx := context.Background() - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - n1, c1 := randNode() - n2, c2 := randNode() - - if err := dserv.Add(ctx, n1); err != nil { - t.Fatal(err) - } - if err := dserv.Add(ctx, n2); err != nil { - t.Fatal(err) - } - - if err := p.Pin(ctx, n1, true); err != nil { - t.Fatal(err) - } - - if err := p.Update(ctx, c1, c2, true); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, c2, "c2 should be pinned now") - assertUnpinned(t, p, c1, "c1 should no longer be pinned") - - if err := p.Update(ctx, c2, c1, false); err != nil { - t.Fatal(err) - } - - assertPinned(t, p, c2, "c2 should be pinned still") - assertPinned(t, p, c1, "c1 should be pinned now") -} From 77c9a1d42aae2e29093004f0cb71a4489f9cc953 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 27 Oct 2020 02:23:14 -0700 Subject: [PATCH 03/29] Import and Export functions. Cid stored as bytes. Revise indexer interface. --- dsindex/indexer.go | 150 +++++++++++++--------- dsindex/indexer_test.go | 143 ++++++++++++++++----- dspinner/pin.go | 167 ++++++++++++++++--------- dspinner/pin_test.go | 270 ++++++++++++++++++++++++++++------------ ipldpinner/pin.go | 3 - 5 files changed, 503 insertions(+), 230 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 85da5e5..f084d45 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -14,9 +14,6 @@ type Indexer interface { // Add adds the a to the an index Add(index, id string) error - // Has determines if a key is in an index - Has(index, id string) (bool, error) - // Delete deletes the specified key from the index. If the key is not in // the datastore, this method returns no error. Delete(index, id string) error @@ -25,12 +22,21 @@ type Indexer interface { // datastore, this method returns no error. DeleteAll(index string) (count int, err error) + // ForEach calls the function for each key in the specified index, until + // there are no more keys, or until the function returns false. If index + // is empty string, then all index names are iterated. + ForEach(index string, fn func(index, id string) bool) error + + // HasKey determines the specified index contains the specified primary key + HasKey(index, id string) (bool, error) + + // HasAny determines if any key is in the specified index. If index is + // empty string, then all indexes are searched. + HasAny(index string) (bool, error) + // Search returns all keys for the given index Search(index string) (ids []string, err error) - // All returns a map of key to secondary index value for all indexed keys - All() (map[string]string, error) - // Synchronize the indexes in this Indexer to match those of the given // Indexer. The indexPath prefix is not synchronized, only the index/key // portion of the indexes. @@ -60,48 +66,71 @@ func (x *indexer) Add(index, id string) error { return x.dstore.Put(key, []byte{}) } -func (x *indexer) Has(index, id string) (bool, error) { - key := ds.NewKey(path.Join(x.indexPath, index, id)) - return x.dstore.Has(key) -} - func (x *indexer) Delete(index, id string) error { - key := ds.NewKey(path.Join(x.indexPath, index, id)) - return x.dstore.Delete(key) + return x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, index, id))) } func (x *indexer) DeleteAll(index string) (int, error) { - ids, err := x.Search(index) + ents, err := x.queryPrefix(path.Join(x.indexPath, index)) if err != nil { return 0, err } - for i := range ids { - err = x.Delete(index, ids[i]) + + for i := range ents { + err = x.dstore.Delete(ds.NewKey(ents[i].Key)) if err != nil { return 0, err } } - return len(ids), nil + + return len(ents), nil } -func (x *indexer) Search(index string) ([]string, error) { - ents, err := x.queryPrefix(path.Join(x.indexPath, index)) - if err != nil { - return nil, err +func (x *indexer) ForEach(index string, fn func(idx, id string) bool) error { + q := query.Query{ + Prefix: path.Join(x.indexPath, index), + KeysOnly: true, } - if len(ents) == 0 { - return nil, nil + results, err := x.dstore.Query(q) + if err != nil { + return err } - ids := make([]string, len(ents)) - for i := range ents { - ids[i] = path.Base(ents[i].Key) + for { + r, ok := results.NextSync() + if !ok { + break + } + if r.Error != nil { + err = r.Error + break + } + + ent := r.Entry + if !fn(path.Base(path.Dir(ent.Key)), path.Base(ent.Key)) { + break + } } - return ids, nil + results.Close() + + return err } -func (x *indexer) All() (map[string]string, error) { - ents, err := x.queryPrefix(x.indexPath) +func (x *indexer) HasKey(index, id string) (bool, error) { + return x.dstore.Has(ds.NewKey(path.Join(x.indexPath, index, id))) +} + +func (x *indexer) HasAny(index string) (bool, error) { + var any bool + err := x.ForEach(index, func(idx, id string) bool { + any = true + return false + }) + return any, err +} + +func (x *indexer) Search(index string) ([]string, error) { + ents, err := x.queryPrefix(path.Join(x.indexPath, index)) if err != nil { return nil, err } @@ -109,55 +138,60 @@ func (x *indexer) All() (map[string]string, error) { return nil, nil } - indexes := make(map[string]string, len(ents)) + ids := make([]string, len(ents)) for i := range ents { - fullPath := ents[i].Key - indexes[path.Base(fullPath)] = path.Base(path.Dir(fullPath)) + ids[i] = path.Base(ents[i].Key) } - - return indexes, nil + return ids, nil } -func (x *indexer) SyncTo(ref Indexer) (changed bool, err error) { - var curAll, refAll map[string]string - - refAll, err = ref.All() +func (x *indexer) SyncTo(ref Indexer) (bool, error) { + // Build reference index map + refs := map[string]string{} + err := ref.ForEach("", func(idx, id string) bool { + refs[id] = idx + return true + }) if err != nil { - return + return false, err } - - curAll, err = x.All() - if err != nil { - return + if len(refs) == 0 { + return false, nil } - for k, v := range refAll { - cv, ok := curAll[k] - if ok && cv == v { - // same in both, so delete from both - delete(curAll, k) - delete(refAll, k) + // Compare current indexes + var delKeys []string + err = x.ForEach("", func(idx, id string) bool { + refIdx, ok := refs[id] + if ok && refIdx == idx { + // same in both; delete from refs, do not add to delKeys + delete(refs, id) + } else { + delKeys = append(delKeys, path.Join(x.indexPath, idx, id)) } + return true + }) + if err != nil { + return false, err } - // What remains in curAll are indexes that no longer exist - for k, v := range curAll { - err = x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, v, k))) + // Items in delKeys are indexes that no longer exist + for i := range delKeys { + err = x.dstore.Delete(ds.NewKey(delKeys[i])) if err != nil { - return + return false, err } } - // What remains in refAll are indexes that need to be added - for k, v := range refAll { + // What remains in refs are indexes that need to be added + for k, v := range refs { err = x.dstore.Put(ds.NewKey(path.Join(x.indexPath, v, k)), nil) if err != nil { - return + return false, err } } - changed = len(refAll) != 0 || len(curAll) != 0 - return + return len(refs) != 0 || len(delKeys) != 0, nil } func (x *indexer) queryPrefix(prefix string) ([]query.Entry, error) { diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go index 93546c4..c8b679b 100644 --- a/dsindex/indexer_test.go +++ b/dsindex/indexer_test.go @@ -34,10 +34,10 @@ func TestAdd(t *testing.T) { } } -func TestHas(t *testing.T) { +func TestHasKey(t *testing.T) { nameIndex := createIndexer() - ok, err := nameIndex.Has("bob", "b1") + ok, err := nameIndex.HasKey("bob", "b1") if err != nil { t.Fatal(err) } @@ -45,7 +45,7 @@ func TestHas(t *testing.T) { t.Fatal("missing index") } - ok, err = nameIndex.Has("bob", "b3") + ok, err = nameIndex.HasKey("bob", "b3") if err != nil { t.Fatal(err) } @@ -54,6 +54,89 @@ func TestHas(t *testing.T) { } } +func TestHasAny(t *testing.T) { + nameIndex := createIndexer() + + ok, err := nameIndex.HasAny("nothere") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } + + for _, idx := range []string{"alice", "bob", ""} { + ok, err = nameIndex.HasAny(idx) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("missing indexe", idx) + } + } + + count, err := nameIndex.DeleteAll("") + if err != nil { + t.Fatal(err) + } + if count != 4 { + t.Fatal("expected 4 deletions") + } + + ok, err = nameIndex.HasAny("") + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("should return false") + } +} + +func TestForEach(t *testing.T) { + nameIndex := createIndexer() + + found := make(map[string]struct{}) + err := nameIndex.ForEach("bob", func(idx, id string) bool { + found[id] = struct{}{} + return true + }) + if err != nil { + t.Fatal(err) + } + + for _, idx := range []string{"b1", "b2"} { + _, ok := found[idx] + if !ok { + t.Fatal("missing index for key", idx) + } + } + + keys := map[string]string{} + err = nameIndex.ForEach("", func(idx, id string) bool { + keys[id] = idx + return true + }) + if err != nil { + t.Fatal(err) + } + if len(keys) != 4 { + t.Fatal("expected 4 keys") + } + + if keys["a1"] != "alice" { + t.Error("expected a1: alice") + } + if keys["b1"] != "bob" { + t.Error("expected b1: bob") + } + if keys["b2"] != "bob" { + t.Error("expected b2: bob") + } + if keys["c1"] != "cathy" { + t.Error("expected c1: cathy") + } +} + func TestSearch(t *testing.T) { nameIndex := createIndexer() @@ -103,7 +186,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } - ok, err := nameIndex.Has("alice", "a1") + ok, err := nameIndex.HasKey("alice", "a1") if err != nil { t.Fatal(err) } @@ -118,29 +201,12 @@ func TestDelete(t *testing.T) { if count != 2 { t.Fatal("wrong deleted count") } - ok, _ = nameIndex.Has("bob", "b1") + ok, _ = nameIndex.HasKey("bob", "b1") if ok { t.Fatal("index not deleted") } } -func TestAll(t *testing.T) { - nameIndex := createIndexer() - - all, err := nameIndex.All() - if err != nil { - t.Fatal(err) - } - if len(all) != 4 { - t.Fatal("wrong number of indexes") - } - for k, v := range all { - if ok, _ := nameIndex.Has(v, k); !ok { - t.Fatal("indexes from all do not match what indexer has") - } - } -} - func TestSyndTo(t *testing.T) { nameIndex := createIndexer() @@ -158,16 +224,31 @@ func TestSyndTo(t *testing.T) { t.Error("change was not indicated") } - refAll, _ := refIndex.All() - syncAll, _ := nameIndex.All() - - if len(syncAll) != len(refAll) { - t.Fatal("wrong number of indexes after sync") + // Create map of id->index in sync target + syncs := map[string]string{} + err = nameIndex.ForEach("", func(idx, id string) bool { + syncs[id] = idx + return true + }) + if err != nil { + t.Fatal(err) } - for k, v := range refAll { - vSync, ok := syncAll[k] - if !ok || v != vSync { - t.Fatal("index", v, "-->", k, "was not synced") + + // Iterate items in sync source and make sure they appear in target + var itemCount int + err = refIndex.ForEach("", func(idx, id string) bool { + itemCount++ + syncIdx, ok := syncs[id] + if !ok || idx != syncIdx { + t.Fatal("index", idx, "-->", id, "was not synced") } + return true + }) + if err != nil { + t.Fatal(err) + } + + if itemCount != len(syncs) { + t.Fatal("different number of items in sync source and target") } } diff --git a/dspinner/pin.go b/dspinner/pin.go index 0973ca2..59d96d2 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -37,6 +37,8 @@ var ( linkDirect, linkRecursive string pinCidIndexPath, pinNameIndexPath string + + pinDatastoreKey = ds.NewKey("/local/pins") ) func init() { @@ -478,24 +480,100 @@ func (p *pinner) loadAllPins() ([]*pin, error) { return pins, nil } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, error) { +// ImportFromIPLDPinner converts pins stored in mdag based storage to pins +// stores in the datastore. Returns a dspinner loaded with the exported pins, +// and a count of the pins imported. +// +// After pins are stored in datastore, the root pin key is deleted to unlink +// the pin data in the DAGService. +func ImportFromIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) defer cancel() - if internal != nil { - ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) - switch err { - case ds.ErrNotFound: - // No more dag storage; load from datastore - case nil: - // Need to convert from dag storage - return convertIPLDToDSPinner(ctx, ipldPinner, dstore, dserv) - default: - return nil, err + ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) + if err != nil { + return nil, 0, err + } + + p := New(dstore, dserv).(*pinner) + + // Save pinned CIDs as new pins in datastore. + rCids, _ := ipldPinner.RecursiveKeys(ctx) + for i := range rCids { + err = p.addPin(rCids[i], ipfspinner.Recursive, "") + if err != nil { + return nil, 0, err + } + } + dCids, _ := ipldPinner.DirectKeys(ctx) + for i := range dCids { + err = p.addPin(dCids[i], ipfspinner.Direct, "") + if err != nil { + return nil, 0, err + } + } + + // Delete root mdag key from datastore to remove old pin storage. + if err = dstore.Delete(pinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot delete old pin state: %v", err) + } + if err = dstore.Sync(pinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot sync old pin state: %v", err) + } + + return p, len(rCids) + len(dCids), nil +} + +// ExportToIPLDPinner exports the pins stored in the datastore by dspinner, and +// imports them into the given internal DAGService. Returns an ipldpinner +// loaded with the exported pins, and a count of the pins exported. +// +// After the pins are stored in the DAGService, the pins and their indexes are +// removed. +func ExportToIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { + p := New(dstore, dserv).(*pinner) + pins, err := p.loadAllPins() + if err != nil { + return nil, 0, fmt.Errorf("cannot load pins: %v", err) + } + + ipldPinner := ipldpinner.New(dstore, dserv, internal) + + seen := cid.NewSet() + for _, pp := range pins { + if seen.Has(pp.cid) { + // multiple pins not support; can only keep one + continue } + seen.Add(pp.cid) + ipldPinner.PinWithMode(pp.cid, pp.mode) + } + + ctx := context.TODO() + + // Save the ipldpinner pins + err = ipldPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + // Remove the dspinner pins and indexes + for _, pp := range pins { + p.removePin(pp) + } + err = p.Flush(ctx) + if err != nil { + return nil, 0, err } + return ipldPinner, seen.Len(), nil +} + +// LoadPinner loads a pinner and its keysets from the given datastore +func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) + defer cancel() + p := New(dstore, dserv).(*pinner) err := p.rebuildIndexes(ctx) if err != nil { @@ -614,10 +692,10 @@ func (p *pinner) Flush(ctx context.Context) error { } } - // TODO: is it necessary to keep a list of added pins to sync? - //if err := p.dstore.Sync(pinKey); err != nil { - // return fmt.Errorf("cannot sync pin state: %v", err) - //} + // Sync pins and indexes + if err := p.dstore.Sync(ds.NewKey(pinKeyPath)); err != nil { + return fmt.Errorf("cannot sync pin state: %v", err) + } return nil } @@ -628,12 +706,18 @@ func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { p.lock.Lock() defer p.lock.Unlock() - ids, _ := p.cidIndex.Search(c.String()) - for i := range ids { - pp, _ := p.loadPin(ids[i]) - if pp != nil && pp.mode == mode { - return // already a pin for this CID with this mode + // TODO: remove his to support multiple pins per CID + switch mode { + case ipfspinner.Recursive: + if p.recursePin.Has(c) { + return // already a recursive pin for this CID + } + case ipfspinner.Direct: + if p.directPin.Has(c) { + return // already a direct pin for this CID } + default: + panic("unrecognized pin mode") } err := p.addPin(c, mode, "") @@ -668,47 +752,12 @@ func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.C return false, nil } -// convertIPLDToDSPinner converts pins stored in mdag based storage to pins -// stores in the datastore. After pins are stored in datastore, then root pin -// key is deleted to unlink the pin data in the mdag store. -func convertIPLDToDSPinner(ctx context.Context, ipldPinner ipfspinner.Pinner, dstore ds.Datastore, dserv ipld.DAGService) (*pinner, error) { - var err error - p := New(dstore, dserv).(*pinner) - - // Save pinned CIDs as new pins in datastore. - rCids, _ := ipldPinner.RecursiveKeys(ctx) - for i := range rCids { - err = p.addPin(rCids[i], ipfspinner.Recursive, "") - if err != nil { - return nil, err - } - } - dCids, _ := ipldPinner.DirectKeys(ctx) - for i := range rCids { - err = p.addPin(dCids[i], ipfspinner.Direct, "") - if err != nil { - return nil, err - } - } - - // Delete root mdag key from datastore to remove old pin storage. - pinDatastoreKey := ds.NewKey("/local/pins") - if err = dstore.Delete(pinDatastoreKey); err != nil { - return nil, fmt.Errorf("cannot delete old pin state: %v", err) - } - if err = dstore.Sync(pinDatastoreKey); err != nil { - return nil, fmt.Errorf("cannot sync old pin state: %v", err) - } - - return p, nil -} - func encodePin(p *pin) ([]byte, error) { var buf bytes.Buffer encoder := cbor.NewMarshaller(&buf) pinData := map[string]interface{}{ "mode": p.mode, - "cid": p.cid.String(), + "cid": p.cid.Bytes(), } // Encode optional fields if p.name != "" { @@ -739,11 +788,11 @@ func decodePin(pid string, data []byte) (*pin, error) { if !ok { return nil, fmt.Errorf("missing cid") } - cidStr, ok := cidData.(string) + cidBytes, ok := cidData.([]byte) if !ok { return nil, fmt.Errorf("invalid pin cid data") } - c, err := cid.Decode(cidStr) + c, err := cid.Cast(cidBytes) if err != nil { return nil, fmt.Errorf("cannot decode pin cid: %v", err) } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 55c2ad7..cbff67a 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -2,6 +2,7 @@ package dspinner import ( "context" + "errors" "io" "testing" "time" @@ -15,7 +16,9 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" ipfspin "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/ipldpinner" util "github.com/ipfs/go-ipfs-util" + ipld "github.com/ipfs/go-ipld-format" ) var rand = util.NewTimeSeededRand() @@ -151,7 +154,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv, dserv) + np, err := LoadPinner(dstore, dserv) if err != nil { t.Fatal(err) } @@ -161,11 +164,33 @@ func TestPinnerBasic(t *testing.T) { // Test recursively pinned assertPinned(t, np, bk, "could not find recursively pinned node") + + ipldPinner, expCount, err := ExportToIPLDPinner(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if expCount != 2 { + t.Fatal("expected 2 exported pins, got", expCount) + } + + assertPinned(t, ipldPinner, ak, "Could not find pinned node!") + assertPinned(t, ipldPinner, bk, "could not find recursively pinned node") + + impPinner, impCount, err := ImportFromIPLDPinner(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if impCount != expCount { + t.Fatal("expected", expCount, "imported pins, got", impCount) + } + + assertPinned(t, impPinner, ak, "Could not find pinned node!") + assertPinned(t, impPinner, bk, "could not find recursively pinned node") } func TestIsPinnedLookup(t *testing.T) { - // We are going to test that lookups work in pins which share - // the same branches. For that we will construct this tree: + // Test that lookups work in pins which share + // the same branches. For that construct this tree: // // A5->A4->A3->A2->A1->A0 // / / @@ -173,12 +198,9 @@ func TestIsPinnedLookup(t *testing.T) { // \ / // C--------------- // - // We will ensure that IsPinned works for all objects both when they + // This ensures that IsPinned works for all objects both when they // are pinned and once they have been unpinned. aBranchLen := 6 - if aBranchLen < 3 { - t.Fatal("set aBranchLen to at least 3") - } ctx := context.Background() dstore := dssync.MutexWrap(ds.NewMapDatastore()) @@ -190,74 +212,10 @@ func TestIsPinnedLookup(t *testing.T) { // TODO does pinner need to share datastore with blockservice? p := New(dstore, dserv) - aNodes := make([]*mdag.ProtoNode, aBranchLen) - aKeys := make([]cid.Cid, aBranchLen) - for i := 0; i < aBranchLen; i++ { - a, _ := randNode() - if i >= 1 { - err := a.AddNodeLink("child", aNodes[i-1]) - if err != nil { - t.Fatal(err) - } - } - - err := dserv.Add(ctx, a) - if err != nil { - t.Fatal(err) - } - //t.Logf("a[%d] is %s", i, ak) - aNodes[i] = a - aKeys[i] = a.Cid() - } - - // Pin A5 recursively - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { - t.Fatal(err) - } - - // Create node B and add A3 as child - b, _ := randNode() - if err := b.AddNodeLink("mychild", aNodes[3]); err != nil { - t.Fatal(err) - } - - // Create C node - c, _ := randNode() - // Add A0 as child of C - if err := c.AddNodeLink("child", aNodes[0]); err != nil { - t.Fatal(err) - } - - // Add C - err := dserv.Add(ctx, c) - if err != nil { - t.Fatal(err) - } - ck := c.Cid() - //t.Logf("C is %s", ck) - - // Add C to B and Add B - if err := b.AddNodeLink("myotherchild", c); err != nil { - t.Fatal(err) - } - err = dserv.Add(ctx, b) + aKeys, bk, ck, err := makeTree(ctx, aBranchLen, dserv, p) if err != nil { t.Fatal(err) } - bk := b.Cid() - //t.Logf("B is %s", bk) - - // Pin C recursively - - if err := p.Pin(ctx, c, true); err != nil { - t.Fatal(err) - } - - // Pin B recursively - - if err := p.Pin(ctx, b, true); err != nil { - t.Fatal(err) - } assertPinned(t, p, aKeys[0], "A0 should be pinned") assertPinned(t, p, aKeys[1], "A1 should be pinned") @@ -265,7 +223,7 @@ func TestIsPinnedLookup(t *testing.T) { assertPinned(t, p, bk, "B should be pinned") // Unpin A5 recursively - if err := p.Unpin(ctx, aKeys[5], true); err != nil { + if err = p.Unpin(ctx, aKeys[5], true); err != nil { t.Fatal(err) } @@ -273,7 +231,7 @@ func TestIsPinnedLookup(t *testing.T) { assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") // Unpin B recursively - if err := p.Unpin(ctx, bk, true); err != nil { + if err = p.Unpin(ctx, bk, true); err != nil { t.Fatal(err) } assertUnpinned(t, p, bk, "B should be unpinned") @@ -389,28 +347,182 @@ func TestPinUpdate(t *testing.T) { n1, c1 := randNode() n2, c2 := randNode() - if err := dserv.Add(ctx, n1); err != nil { + err := dserv.Add(ctx, n1) + if err != nil { t.Fatal(err) } - if err := dserv.Add(ctx, n2); err != nil { + if err = dserv.Add(ctx, n2); err != nil { t.Fatal(err) } - if err := p.Pin(ctx, n1, true); err != nil { + if err = p.Pin(ctx, n1, true); err != nil { t.Fatal(err) } - if err := p.Update(ctx, c1, c2, true); err != nil { + if err = p.Update(ctx, c1, c2, true); err != nil { t.Fatal(err) } assertPinned(t, p, c2, "c2 should be pinned now") assertUnpinned(t, p, c1, "c1 should no longer be pinned") - if err := p.Update(ctx, c2, c1, false); err != nil { + if err = p.Update(ctx, c2, c1, false); err != nil { t.Fatal(err) } assertPinned(t, p, c2, "c2 should be pinned still") assertPinned(t, p, c1, "c1 should be pinned now") } + +func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { + if aBranchLen < 3 { + err = errors.New("set aBranchLen to at least 3") + return + } + + aNodes := make([]*mdag.ProtoNode, aBranchLen) + aKeys = make([]cid.Cid, aBranchLen) + for i := 0; i < aBranchLen; i++ { + a, _ := randNode() + if i >= 1 { + if err = a.AddNodeLink("child", aNodes[i-1]); err != nil { + return + } + } + + if err = dserv.Add(ctx, a); err != nil { + return + } + aNodes[i] = a + aKeys[i] = a.Cid() + } + + // Pin last A recursively + if err = p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + return + } + + // Create node B and add A3 as child + b, _ := randNode() + if err = b.AddNodeLink("mychild", aNodes[3]); err != nil { + return + } + + // Create C node + c, _ := randNode() + // Add A0 as child of C + if err = c.AddNodeLink("child", aNodes[0]); err != nil { + return + } + + // Add C + if err = dserv.Add(ctx, c); err != nil { + return + } + ck = c.Cid() + + // Add C to B and Add B + if err = b.AddNodeLink("myotherchild", c); err != nil { + return + } + if err = dserv.Add(ctx, b); err != nil { + return + } + bk = b.Cid() + + // Pin C recursively + if err = p.Pin(ctx, c, true); err != nil { + return + } + + // Pin B recursively + if err = p.Pin(ctx, b, true); err != nil { + return + } + + if err = p.Flush(ctx); err != nil { + return + } + + return +} + +func BenchmarkPinDSPinner(b *testing.B) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := pinMany(ctx, i, dserv, p) + if err != nil { + panic(err.Error()) + } + } +} + +func BenchmarkPinIPLDPinner(b *testing.B) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := ipldpinner.New(dstore, dserv, dserv) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := pinMany(ctx, i, dserv, p) + if err != nil { + panic(err.Error()) + } + } +} + +func BenchmarkIsPinnedDSPinner(b *testing.B) { + ctx := context.Background() + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + p := New(dstore, dserv) + + keys, err := pinMany(ctx, b.N, dserv, p) + if err != nil { + panic(err.Error()) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _, err := p.IsPinned(ctx, keys[i]) + if err != nil { + panic(err.Error()) + } + + } +} + +func pinMany(ctx context.Context, count int, dserv ipld.DAGService, p ipfspin.Pinner) ([]cid.Cid, error) { + keys := make([]cid.Cid, count) + for i := 0; i < count; i++ { + a, _ := randNode() + err := dserv.Add(ctx, a) + if err != nil { + return nil, err + } + if err = p.Pin(ctx, a, true); err != nil { + return nil, err + } + keys[i] = a.Cid() + } + p.Flush(context.Background()) + return keys, nil +} diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go index 6f28430..b2bda9d 100644 --- a/ipldpinner/pin.go +++ b/ipldpinner/pin.go @@ -316,9 +316,6 @@ func cidSetWithValues(cids []cid.Cid) *cid.Set { func LoadPinner(dstore ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { rootKey, err := dstore.Get(pinDatastoreKey) if err != nil { - if err == ds.ErrNotFound { - return nil, err - } return nil, fmt.Errorf("cannot load pin state: %v", err) } rootCid, err := cid.Cast(rootKey) From 5398bb2c24cccaa2e164360b487f8dd914de7290 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 27 Oct 2020 09:45:47 -0700 Subject: [PATCH 04/29] add name index --- dspinner/pin.go | 55 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 59d96d2..426fd07 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -68,7 +68,8 @@ type pinner struct { dserv ipld.DAGService dstore ds.Datastore - cidIndex dsindex.Indexer + cidIndex dsindex.Indexer + nameIndex dsindex.Indexer } var _ ipfspinner.Pinner = (*pinner)(nil) @@ -105,6 +106,7 @@ type syncDAGService interface { func New(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { return &pinner{ cidIndex: dsindex.New(dstore, pinCidIndexPath), + nameIndex: dsindex.New(dstore, pinNameIndexPath), dserv: serv, dstore: dstore, directPin: cid.NewSet(), @@ -183,10 +185,21 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { return fmt.Errorf("could not add pin cid index: %v", err) } + if name != "" { + // Store name index + err = p.nameIndex.Add(name, pp.id) + if err != nil { + return fmt.Errorf("could not add pin name index: %v", err) + } + } + // Store the pin err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { p.cidIndex.Delete(c.String(), pp.id) + if name != "" { + p.nameIndex.Delete(name, pp.id) + } return err } @@ -201,24 +214,32 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { return nil } -func (p *pinner) removePin(pin *pin) error { +func (p *pinner) removePin(pp *pin) error { // Remove pin from datastore - err := p.dstore.Delete(pin.dsKey()) + err := p.dstore.Delete(pp.dsKey()) if err != nil { return err } // Remove cid index from datastore - err = p.cidIndex.Delete(pin.cid.String(), pin.id) + err = p.cidIndex.Delete(pp.cid.String(), pp.id) if err != nil { return err } + if pp.name != "" { + // Remove name index from datastore + err = p.nameIndex.Delete(pp.name, pp.id) + if err != nil { + return err + } + } + // Update cache - switch pin.mode { + switch pp.mode { case ipfspinner.Recursive: - p.recursePin.Remove(pin.cid) + p.recursePin.Remove(pp.cid) case ipfspinner.Direct: - p.directPin.Remove(pin.cid) + p.directPin.Remove(pp.cid) } return nil @@ -249,6 +270,7 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { return err } if !ok { + p.cidIndex.DeleteAll(c.String()) log.Error("found CID index with missing pin") } return nil @@ -598,8 +620,14 @@ func (p *pinner) rebuildIndexes(ctx context.Context) error { // Build temporary in-memory CID index from pins dstoreMem := ds.NewMapDatastore() tmpCidIndex := dsindex.New(dstoreMem, pinCidIndexPath) + tmpNameIndex := dsindex.New(dstoreMem, pinNameIndexPath) + var hasNames bool for _, pp := range pins { tmpCidIndex.Add(pp.cid.String(), pp.id) + if pp.name != "" { + tmpNameIndex.Add(pp.name, pp.id) + hasNames = true + } // Build up cache if pp.mode == ipfspinner.Recursive { @@ -614,10 +642,19 @@ func (p *pinner) rebuildIndexes(ctx context.Context) error { // and writing secondary index. changed, err := p.cidIndex.SyncTo(tmpCidIndex) if err != nil { - return fmt.Errorf("cannot sync indexes: %v", err) + return fmt.Errorf("cannot sync cid indexes: %v", err) } if changed { - log.Error("invalid indexes detected - rebuilt") + log.Error("invalid cid indexes detected - rebuilt") + } + if hasNames { + changed, err = p.nameIndex.SyncTo(tmpNameIndex) + if err != nil { + return fmt.Errorf("cannot sync name indexes: %v", err) + } + if changed { + log.Error("invalid name indexes detected - rebuilt") + } } return nil From 922fab7ac997671546ae5e582dbd25e4d6d07f34 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 27 Oct 2020 15:46:21 -0700 Subject: [PATCH 05/29] add benchmarks --- dspinner/pin_test.go | 171 ++++++++++++++++++++++++++++++++----------- 1 file changed, 128 insertions(+), 43 deletions(-) diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index cbff67a..1438169 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -447,82 +447,167 @@ func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfs return } -func BenchmarkPinDSPinner(b *testing.B) { +func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + nodes := make([]ipld.Node, count) + for i := 0; i < count; i++ { + n, _ := randNode() + err := dserv.Add(ctx, n) + if err != nil { + panic(err) + } + nodes[i] = n + } + return nodes +} - b.ResetTimer() +func pinNodes(nodes []ipld.Node, p ipfspin.Pinner) { + ctx := context.Background() + var err error - for i := 0; i < b.N; i++ { - _, err := pinMany(ctx, i, dserv, p) + for i := range nodes { + err = p.Pin(ctx, nodes[i], true) if err != nil { - panic(err.Error()) + panic(err) } } + err = p.Flush(ctx) + if err != nil { + panic(err) + } } -func BenchmarkPinIPLDPinner(b *testing.B) { +func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) { ctx := context.Background() + var err error + + for i := range nodes { + err = p.Unpin(ctx, nodes[i].Cid(), true) + if err != nil { + panic(err) + } + } + err = p.Flush(ctx) + if err != nil { + panic(err) + } +} + +func makeStore() (ds.Datastore, ipld.DAGService) { dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) - dserv := mdag.NewDAGService(bserv) - p := ipldpinner.New(dstore, dserv, dserv) + return dstore, dserv +} +func BenchmarkPinDS(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkPin(b, pinner, dserv) +} + +func BenchmarkPinIPLD(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkPin(b, pinner, dserv) +} + +func benchmarkPin(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx := context.Background() + nodes := makeNodes(b.N, dserv) b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := pinMany(ctx, i, dserv, p) + err := pinner.Pin(ctx, nodes[i], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) if err != nil { - panic(err.Error()) + panic(err) } } } -func BenchmarkIsPinnedDSPinner(b *testing.B) { - ctx := context.Background() - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) +func BenchmarkUnpinDS(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkUnpin(b, pinner, dserv) +} - dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) +func BenchmarkUnpinIPLD(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkUnpin(b, pinner, dserv) +} - keys, err := pinMany(ctx, b.N, dserv, p) - if err != nil { - panic(err.Error()) - } +func benchmarkUnpin(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx := context.Background() + nodes := makeNodes(b.N, dserv) + pinNodes(nodes, pinner) b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := p.IsPinned(ctx, keys[i]) + err := pinner.Unpin(ctx, nodes[i].Cid(), true) if err != nil { - panic(err.Error()) + panic(err) } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + } +} + +func BenchmarkPinAllDS(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkPinAll(b, pinner, dserv) +} + +func BenchmarkPinAllIPLD(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkPinAll(b, pinner, dserv) +} + +func benchmarkPinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { + nodes := makeNodes(b.N, dserv) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + pinNodes(nodes, pinner) + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() } } -func pinMany(ctx context.Context, count int, dserv ipld.DAGService, p ipfspin.Pinner) ([]cid.Cid, error) { - keys := make([]cid.Cid, count) - for i := 0; i < count; i++ { - a, _ := randNode() - err := dserv.Add(ctx, a) - if err != nil { - return nil, err - } - if err = p.Pin(ctx, a, true); err != nil { - return nil, err - } - keys[i] = a.Cid() +func BenchmarkUnpinAllDS(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkUnpinAll(b, pinner, dserv) +} + +func BenchmarkUnpinAllIPLD(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkUnpinAll(b, pinner, dserv) +} + +func benchmarkUnpinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { + nodes := makeNodes(b.N, dserv) + pinNodes(nodes, pinner) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + unpinNodes(nodes, pinner) + + b.StopTimer() + pinNodes(nodes, pinner) + b.StartTimer() } - p.Flush(context.Background()) - return keys, nil } From fdf37b10dd048f8d15ad8e187d56e84a1b4c05f1 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 27 Oct 2020 17:45:01 -0700 Subject: [PATCH 06/29] Use dirty flag to determine when to rebuild indexes And and improve benchmarks --- dspinner/pin.go | 72 ++++++++++++++++++++++++++++++++------------ dspinner/pin_test.go | 54 ++++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 426fd07..6c1e0cd 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -27,6 +27,7 @@ const ( pinKeyPath = "/.pins/pin" indexKeyPath = "/.pins/index" + dirtyKeyPath = "/.pins/state/dirty" ) var ( @@ -39,6 +40,8 @@ var ( pinCidIndexPath, pinNameIndexPath string pinDatastoreKey = ds.NewKey("/local/pins") + + dirtyKey = ds.NewKey(dirtyKeyPath) ) func init() { @@ -70,6 +73,8 @@ type pinner struct { cidIndex dsindex.Indexer nameIndex dsindex.Indexer + + dirty bool } var _ ipfspinner.Pinner = (*pinner)(nil) @@ -179,6 +184,8 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { return fmt.Errorf("could not encode pin: %v", err) } + p.setDirty(true) + // Store CID index err = p.cidIndex.Add(c.String(), pp.id) if err != nil { @@ -215,6 +222,8 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { } func (p *pinner) removePin(pp *pin) error { + p.setDirty(true) + // Remove pin from datastore err := p.dstore.Delete(pp.dsKey()) if err != nil { @@ -270,6 +279,7 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { return err } if !ok { + p.setDirty(true) p.cidIndex.DeleteAll(c.String()) log.Error("found CID index with missing pin") } @@ -597,9 +607,33 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, defer cancel() p := New(dstore, dserv).(*pinner) - err := p.rebuildIndexes(ctx) + + pins, err := p.loadAllPins() + if err != nil { + return nil, fmt.Errorf("cannot load pins: %v", err) + } + for _, pp := range pins { + // Build up cache + if pp.mode == ipfspinner.Recursive { + p.recursePin.Add(pp.cid) + } else if pp.mode == ipfspinner.Direct { + p.directPin.Add(pp.cid) + } + } + + data, err := dstore.Get(dirtyKey) if err != nil { - return nil, fmt.Errorf("cannot rebuild indexes: %v", err) + if err == ds.ErrNotFound { + return p, nil + } + return nil, fmt.Errorf("cannot load dirty flag: %v", err) + } + if data[0] == 1 { + p.dirty = true + err := p.rebuildIndexes(ctx, pins) + if err != nil { + return nil, fmt.Errorf("cannot rebuild indexes: %v", err) + } } return p, nil @@ -608,15 +642,7 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, // rebuildIndexes uses the stored pins to rebuild secondary indexes. This // resolves any discrepancy between secondary indexes and pins that could // result from a program termination between saving the two. -func (p *pinner) rebuildIndexes(ctx context.Context) error { - pins, err := p.loadAllPins() - if err != nil { - return fmt.Errorf("cannot load pins: %v", err) - } - - p.directPin = cid.NewSet() - p.recursePin = cid.NewSet() - +func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { // Build temporary in-memory CID index from pins dstoreMem := ds.NewMapDatastore() tmpCidIndex := dsindex.New(dstoreMem, pinCidIndexPath) @@ -628,13 +654,6 @@ func (p *pinner) rebuildIndexes(ctx context.Context) error { tmpNameIndex.Add(pp.name, pp.id) hasNames = true } - - // Build up cache - if pp.mode == ipfspinner.Recursive { - p.recursePin.Add(pp.cid) - } else if pp.mode == ipfspinner.Direct { - p.directPin.Add(pp.cid) - } } // Sync the CID index to what was build from pins. This fixes any invalid @@ -657,7 +676,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context) error { } } - return nil + return p.Flush(ctx) } // DirectKeys returns a slice containing the directly pinned keys @@ -734,6 +753,8 @@ func (p *pinner) Flush(ctx context.Context) error { return fmt.Errorf("cannot sync pin state: %v", err) } + p.setDirty(false) + return nil } @@ -869,3 +890,16 @@ func decodePin(pid string, data []byte) (*pin, error) { return p, nil } + +func (p *pinner) setDirty(dirty bool) { + if p.dirty == dirty { + return + } + p.dirty = dirty + data := []byte{0} + if dirty { + data[0] = 1 + } + p.dstore.Put(dirtyKey, data) + p.dstore.Sync(dirtyKey) +} diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 1438169..90906e3 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -461,12 +461,12 @@ func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { return nodes } -func pinNodes(nodes []ipld.Node, p ipfspin.Pinner) { +func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) { ctx := context.Background() var err error for i := range nodes { - err = p.Pin(ctx, nodes[i], true) + err = p.Pin(ctx, nodes[i], recursive) if err != nil { panic(err) } @@ -501,6 +501,36 @@ func makeStore() (ds.Datastore, ipld.DAGService) { return dstore, dserv } +func BenchmarkLoadRebuild(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + + nodes := makeNodes(1024, dserv) + pinNodes(nodes, pinner, true) + + b.Run("RebuildTrue", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dstore.Put(dirtyKey, []byte{1}) + + _, err := LoadPinner(dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) + + b.Run("RebuildFalse", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dstore.Put(dirtyKey, []byte{0}) + + _, err := LoadPinner(dstore, dserv) + if err != nil { + panic(err.Error()) + } + } + }) +} + func BenchmarkPinDS(b *testing.B) { dstore, dserv := makeStore() pinner := New(dstore, dserv) @@ -545,7 +575,7 @@ func BenchmarkUnpinIPLD(b *testing.B) { func benchmarkUnpin(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { ctx := context.Background() nodes := makeNodes(b.N, dserv) - pinNodes(nodes, pinner) + pinNodes(nodes, pinner, true) b.ResetTimer() @@ -578,7 +608,7 @@ func benchmarkPinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) b.ResetTimer() for i := 0; i < b.N; i++ { - pinNodes(nodes, pinner) + pinNodes(nodes, pinner, true) b.StopTimer() unpinNodes(nodes, pinner) @@ -587,27 +617,21 @@ func benchmarkPinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) } func BenchmarkUnpinAllDS(b *testing.B) { + // There is no point in benchmarking this for the IPLD pinner, because it + // amounts to saving the root internal node with no data; so it ismostly a + // no-op. dstore, dserv := makeStore() pinner := New(dstore, dserv) - benchmarkUnpinAll(b, pinner, dserv) -} - -func BenchmarkUnpinAllIPLD(b *testing.B) { - dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkUnpinAll(b, pinner, dserv) -} -func benchmarkUnpinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { nodes := makeNodes(b.N, dserv) - pinNodes(nodes, pinner) + pinNodes(nodes, pinner, true) b.ResetTimer() for i := 0; i < b.N; i++ { unpinNodes(nodes, pinner) b.StopTimer() - pinNodes(nodes, pinner) + pinNodes(nodes, pinner, true) b.StartTimer() } } From a2720f1dbde24ed6ef93836a27b8073bcf08b031 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 28 Oct 2020 10:25:29 -0700 Subject: [PATCH 07/29] Fix benchmarks --- dspinner/pin_test.go | 158 ++++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 71 deletions(-) diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 90906e3..71697b4 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -3,6 +3,7 @@ package dspinner import ( "context" "errors" + "fmt" "io" "testing" "time" @@ -501,6 +502,9 @@ func makeStore() (ds.Datastore, ipld.DAGService) { return dstore, dserv } +// BenchmarkLoadRebuild loads a pinner that has some number of saved pins, and +// compares the load time when rebuilding indexes to loading without rebuilding +// indexes. func BenchmarkLoadRebuild(b *testing.B) { dstore, dserv := makeStore() pinner := New(dstore, dserv) @@ -531,80 +535,112 @@ func BenchmarkLoadRebuild(b *testing.B) { }) } -func BenchmarkPinDS(b *testing.B) { - dstore, dserv := makeStore() - pinner := New(dstore, dserv) - benchmarkPin(b, pinner, dserv) -} - -func BenchmarkPinIPLD(b *testing.B) { - dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkPin(b, pinner, dserv) +// BenchmarkPinSeries demonstrates creating individual pins. Each run in the +// series shows performance for a larger number of individual pins. +func BenchmarkPinSeries(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkPin(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkPin(b, count, pinner, dserv) + }) + } } -func benchmarkPin(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { +func benchmarkPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { ctx := context.Background() - nodes := makeNodes(b.N, dserv) + nodes := makeNodes(count, dserv) b.ResetTimer() for i := 0; i < b.N; i++ { - err := pinner.Pin(ctx, nodes[i], true) - if err != nil { - panic(err) - } - err = pinner.Flush(ctx) - if err != nil { - panic(err) + // Pin all the nodes one at a time. + for j := range nodes { + err := pinner.Pin(ctx, nodes[j], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } } - } -} -func BenchmarkUnpinDS(b *testing.B) { - dstore, dserv := makeStore() - pinner := New(dstore, dserv) - benchmarkUnpin(b, pinner, dserv) + // Unpin all nodes so that they can be pinned next iter. + b.StopTimer() + unpinNodes(nodes, pinner) + b.StartTimer() + } } -func BenchmarkUnpinIPLD(b *testing.B) { - dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkUnpin(b, pinner, dserv) +// BenchmarkUnpinSeries demonstrates unpinning individual pins. Each run in the +// series shows performance for a larger number of individual unpins. +func BenchmarkUnpinSeries(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkPin(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("UninIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkPin(b, count, pinner, dserv) + }) + } } -func benchmarkUnpin(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { +func benchmarkUnpin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { ctx := context.Background() - nodes := makeNodes(b.N, dserv) + nodes := makeNodes(count, dserv) pinNodes(nodes, pinner, true) - b.ResetTimer() for i := 0; i < b.N; i++ { - err := pinner.Unpin(ctx, nodes[i].Cid(), true) - if err != nil { - panic(err) - } - err = pinner.Flush(ctx) - if err != nil { - panic(err) + for j := range nodes { + // Unpin nodes one at a time. + err := pinner.Unpin(ctx, nodes[j].Cid(), true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } } + // Pin all nodes so that they can be unpinned next iter. + b.StopTimer() + pinNodes(nodes, pinner, true) + b.StartTimer() } } -func BenchmarkPinAllDS(b *testing.B) { - dstore, dserv := makeStore() - pinner := New(dstore, dserv) - benchmarkPinAll(b, pinner, dserv) -} - -func BenchmarkPinAllIPLD(b *testing.B) { - dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkPinAll(b, pinner, dserv) +// BenchmarkPinAllSeries shows times to pin all nodes with only one Flush at +// the end. +func BenchmarkPinAllSeries(b *testing.B) { + for count := 128; count < 16386; count <<= 1 { + b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + benchmarkPinAll(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinAllIPLD-", count), func(b *testing.B) { + dstore, dserv := makeStore() + pinner := ipldpinner.New(dstore, dserv, dserv) + benchmarkPinAll(b, count, pinner, dserv) + }) + } } -func benchmarkPinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) { - nodes := makeNodes(b.N, dserv) +func benchmarkPinAll(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + nodes := makeNodes(count, dserv) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -615,23 +651,3 @@ func benchmarkPinAll(b *testing.B, pinner ipfspin.Pinner, dserv ipld.DAGService) b.StartTimer() } } - -func BenchmarkUnpinAllDS(b *testing.B) { - // There is no point in benchmarking this for the IPLD pinner, because it - // amounts to saving the root internal node with no data; so it ismostly a - // no-op. - dstore, dserv := makeStore() - pinner := New(dstore, dserv) - - nodes := makeNodes(b.N, dserv) - pinNodes(nodes, pinner, true) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - unpinNodes(nodes, pinner) - - b.StopTimer() - pinNodes(nodes, pinner, true) - b.StartTimer() - } -} From 37293b66fa3685f978ab90809329133884fcce0c Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 28 Oct 2020 19:04:25 -0700 Subject: [PATCH 08/29] Do not keep pinned CID sets in memory (no-cache implementation) - Keep separate recursive and direct CID indexes. This alows searching for a direct or recursive CIDs without having to load pins to check the mode. - Only load pins if dirty flag indicates index repair may be needed - Improved benchmarks --- dspinner/pin.go | 373 ++++++++++++++++++++++++++++++------------- dspinner/pin_test.go | 87 ++++++++-- go.mod | 3 +- go.sum | 34 +++- 4 files changed, 370 insertions(+), 127 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 6c1e0cd..b40edaa 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -23,7 +23,8 @@ import ( ) const ( - loadTimeout = 5 * time.Second + loadTimeout = 5 * time.Second + rebuildTimeout = 2 * time.Minute pinKeyPath = "/.pins/pin" indexKeyPath = "/.pins/index" @@ -36,8 +37,11 @@ var ( log = logging.Logger("pin") - linkDirect, linkRecursive string - pinCidIndexPath, pinNameIndexPath string + linkDirect, linkRecursive string + + pinCidDIndexPath string + pinCidRIndexPath string + pinNameIndexPath string pinDatastoreKey = ds.NewKey("/local/pins") @@ -57,21 +61,20 @@ func init() { } linkRecursive = recursiveStr - pinCidIndexPath = path.Join(indexKeyPath, "cidindex") - pinNameIndexPath = path.Join(indexKeyPath, "nameindex") + pinCidRIndexPath = path.Join(indexKeyPath, "cidRindex") + pinCidDIndexPath = path.Join(indexKeyPath, "cidDindex") + pinNameIndexPath = path.Join(indexKeyPath, "nameIndex") } // pinner implements the Pinner interface type pinner struct { lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - dserv ipld.DAGService dstore ds.Datastore - cidIndex dsindex.Indexer + cidDIndex dsindex.Indexer + cidRIndex dsindex.Indexer nameIndex dsindex.Indexer dirty bool @@ -110,12 +113,11 @@ type syncDAGService interface { // New creates a new pinner using the given datastore as a backend func New(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { return &pinner{ - cidIndex: dsindex.New(dstore, pinCidIndexPath), - nameIndex: dsindex.New(dstore, pinNameIndexPath), - dserv: serv, - dstore: dstore, - directPin: cid.NewSet(), - recursePin: cid.NewSet(), + cidDIndex: dsindex.New(dstore, pinCidDIndexPath), + cidRIndex: dsindex.New(dstore, pinCidRIndexPath), + nameIndex: dsindex.New(dstore, pinNameIndexPath), + dserv: serv, + dstore: dstore, } } @@ -127,33 +129,50 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { } c := node.Cid() + cidStr := c.String() p.lock.Lock() defer p.lock.Unlock() if recurse { - if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Recursive) { + found, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return err + } + if found { return nil } - p.lock.Unlock() // temporary unlock to fetch the entire graph - err := mdag.FetchGraph(ctx, c, p.dserv) + p.lock.Unlock() + // Fetch graph to ensure that any children of the pinned node are in + // the graph. + err = mdag.FetchGraph(ctx, c, p.dserv) p.lock.Lock() + if err != nil { return err } - if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Recursive) { + found, err = p.cidRIndex.HasAny(cidStr) + if err != nil { + return err + } + if found { return nil } // TODO: remove this to support multiple pins per CID - if p.isPinnedWithTypeBool(ctx, c, ipfspinner.Direct) { + found, err = p.cidDIndex.HasAny(cidStr) + if err != nil { + return err + } + if found { ok, _ := p.removePinsForCid(c, ipfspinner.Direct) if !ok { - // Fix cache - p.directPin.Remove(c) + // Fix index; remove index for pin that does not exist + p.cidDIndex.DeleteAll(c.String()) + log.Error("found cid index for direct pin that does not exist") } } @@ -162,8 +181,12 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return err } } else { - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) + found, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return err + } + if found { + return fmt.Errorf("%s already pinned recursively", cidStr) } err = p.addPin(c, ipfspinner.Direct, "") @@ -187,7 +210,14 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { p.setDirty(true) // Store CID index - err = p.cidIndex.Add(c.String(), pp.id) + switch mode { + case ipfspinner.Recursive: + err = p.cidRIndex.Add(c.String(), pp.id) + case ipfspinner.Direct: + err = p.cidDIndex.Add(c.String(), pp.id) + default: + panic("pin mode must be recursive or direct") + } if err != nil { return fmt.Errorf("could not add pin cid index: %v", err) } @@ -203,21 +233,17 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { // Store the pin err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { - p.cidIndex.Delete(c.String(), pp.id) + if mode == ipfspinner.Recursive { + p.cidRIndex.Delete(c.String(), pp.id) + } else { + p.cidDIndex.Delete(c.String(), pp.id) + } if name != "" { p.nameIndex.Delete(name, pp.id) } return err } - // Update cache - switch mode { - case ipfspinner.Recursive: - p.recursePin.Add(c) - case ipfspinner.Direct: - p.directPin.Add(c) - } - return nil } @@ -230,7 +256,11 @@ func (p *pinner) removePin(pp *pin) error { return err } // Remove cid index from datastore - err = p.cidIndex.Delete(pp.cid.String(), pp.id) + if pp.mode == ipfspinner.Recursive { + err = p.cidRIndex.Delete(pp.cid.String(), pp.id) + } else { + err = p.cidDIndex.Delete(pp.cid.String(), pp.id) + } if err != nil { return err } @@ -243,14 +273,6 @@ func (p *pinner) removePin(pp *pin) error { } } - // Update cache - switch pp.mode { - case ipfspinner.Recursive: - p.recursePin.Remove(pp.cid) - case ipfspinner.Direct: - p.directPin.Remove(pp.cid) - } - return nil } @@ -266,23 +288,38 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } matches := p.Ls(matchSpec) */ - if p.recursePin.Has(c) { + cidStr := c.String() + has, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return err + } + var ok bool + + if has { if !recursive { return fmt.Errorf("%s is pinned recursively", c) } - } else if !p.directPin.Has(c) { - return ErrNotPinned + } else { + has, err = p.cidDIndex.HasAny(cidStr) + if err != nil { + return err + } + if !has { + return ErrNotPinned + } } - ok, err := p.removePinsForCid(c, ipfspinner.Any) + ok, err = p.removePinsForCid(c, ipfspinner.Any) if err != nil { return err } if !ok { p.setDirty(true) - p.cidIndex.DeleteAll(c.String()) + p.cidRIndex.DeleteAll(cidStr) + p.cidDIndex.DeleteAll(cidStr) log.Error("found CID index with missing pin") } + return nil } @@ -302,20 +339,24 @@ func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne return p.isPinnedWithType(ctx, c, mode) } -func (p *pinner) isPinnedWithTypeBool(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) bool { - _, found, _ := p.isPinnedWithType(ctx, c, mode) - return found -} - func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { + cidStr := c.String() switch mode { case ipfspinner.Recursive: - if p.recursePin.Has(c) { + has, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return "", false, err + } + if has { return linkRecursive, true, nil } return "", false, nil case ipfspinner.Direct: - if p.directPin.Has(c) { + has, err := p.cidDIndex.HasAny(cidStr) + if err != nil { + return "", false, err + } + if has { return linkDirect, true, nil } return "", false, nil @@ -323,10 +364,18 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne return "", false, nil case ipfspinner.Indirect: case ipfspinner.Any: - if p.recursePin.Has(c) { + has, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return "", false, err + } + if has { return linkRecursive, true, nil } - if p.directPin.Has(c) { + has, err = p.cidDIndex.HasAny(cidStr) + if err != nil { + return "", false, err + } + if has { return linkDirect, true, nil } default: @@ -339,15 +388,36 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne // Default is Indirect visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err + + // No index for given CID, so search children of all recursive pinned CIDs + var has bool + var rc cid.Cid + var e error + err := p.cidRIndex.ForEach("", func(idx, id string) bool { + rc, e = cid.Decode(idx) + if e != nil { + return false + } + has, e = hasChild(ctx, p.dserv, rc, c, visitedSet.Visit) + if e != nil { + return false } if has { - return rc.String(), true, nil + return false } + return true + }) + if err != nil { + return "", false, err } + if e != nil { + return "", false, e + } + + if has { + return rc.String(), true, nil + } + return "", false, nil } @@ -363,12 +433,23 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn // First check for non-Indirect pins directly for _, c := range cids { - if p.recursePin.Has(c) { + cidStr := c.String() + has, err := p.cidRIndex.HasAny(cidStr) + if err != nil { + return nil, err + } + if has { pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) } else { - toCheck.Add(c) + has, err = p.cidDIndex.HasAny(cidStr) + if err != nil { + return nil, err + } + if has { + pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct}) + } else { + toCheck.Add(c) + } } } @@ -400,14 +481,27 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn return nil } - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) - if err != nil { - return nil, err + var e error + err := p.cidRIndex.ForEach("", func(idx, id string) bool { + var rk cid.Cid + rk, e = cid.Decode(idx) + if e != nil { + return false + } + e = checkChildren(rk, rk) + if e != nil { + return false } if toCheck.Len() == 0 { - break + return false } + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e } // Anything left in toCheck is not pinned @@ -427,14 +521,7 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { // Check cache to see if CID is pinned switch mode { - case ipfspinner.Direct: - if !p.directPin.Has(c) { - return - } - case ipfspinner.Recursive: - if !p.recursePin.Has(c) { - return - } + case ipfspinner.Direct, ipfspinner.Recursive: default: // programmer error, panic OK panic("unrecognized pin type") @@ -446,7 +533,24 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { // removePinsForCid removes all pins for a cid that have the specified mode. func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) { // Search for pins by CID - ids, err := p.cidIndex.Search(c.String()) + var ids []string + var err error + switch mode { + case ipfspinner.Recursive: + ids, err = p.cidRIndex.Search(c.String()) + case ipfspinner.Direct: + ids, err = p.cidDIndex.Search(c.String()) + case ipfspinner.Any: + cidStr := c.String() + ids, err = p.cidRIndex.Search(cidStr) + dIds, dErr := p.cidDIndex.Search(cidStr) + if dErr != nil && err == nil { + err = dErr + } + if len(dIds) != 0 { + ids = append(ids, dIds...) + } + } if err != nil { return false, err } @@ -519,9 +623,6 @@ func (p *pinner) loadAllPins() ([]*pin, error) { // After pins are stored in datastore, the root pin key is deleted to unlink // the pin data in the DAGService. func ImportFromIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { - ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) - defer cancel() - ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) if err != nil { return nil, 0, err @@ -529,6 +630,9 @@ func ImportFromIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal i p := New(dstore, dserv).(*pinner) + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) + defer cancel() + // Save pinned CIDs as new pins in datastore. rCids, _ := ipldPinner.RecursiveKeys(ctx) for i := range rCids { @@ -603,24 +707,8 @@ func ExportToIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipl // LoadPinner loads a pinner and its keysets from the given datastore func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { - ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) - defer cancel() - p := New(dstore, dserv).(*pinner) - pins, err := p.loadAllPins() - if err != nil { - return nil, fmt.Errorf("cannot load pins: %v", err) - } - for _, pp := range pins { - // Build up cache - if pp.mode == ipfspinner.Recursive { - p.recursePin.Add(pp.cid) - } else if pp.mode == ipfspinner.Direct { - p.directPin.Add(pp.cid) - } - } - data, err := dstore.Get(dirtyKey) if err != nil { if err == ds.ErrNotFound { @@ -630,7 +718,16 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, } if data[0] == 1 { p.dirty = true - err := p.rebuildIndexes(ctx, pins) + + pins, err := p.loadAllPins() + if err != nil { + return nil, fmt.Errorf("cannot load pins: %v", err) + } + + ctx, cancel := context.WithTimeout(context.TODO(), rebuildTimeout) + defer cancel() + + err = p.rebuildIndexes(ctx, pins) if err != nil { return nil, fmt.Errorf("cannot rebuild indexes: %v", err) } @@ -645,11 +742,16 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { // Build temporary in-memory CID index from pins dstoreMem := ds.NewMapDatastore() - tmpCidIndex := dsindex.New(dstoreMem, pinCidIndexPath) + tmpCidDIndex := dsindex.New(dstoreMem, pinCidDIndexPath) + tmpCidRIndex := dsindex.New(dstoreMem, pinCidRIndexPath) tmpNameIndex := dsindex.New(dstoreMem, pinNameIndexPath) var hasNames bool for _, pp := range pins { - tmpCidIndex.Add(pp.cid.String(), pp.id) + if pp.mode == ipfspinner.Recursive { + tmpCidRIndex.Add(pp.cid.String(), pp.id) + } else if pp.mode == ipfspinner.Direct { + tmpCidDIndex.Add(pp.cid.String(), pp.id) + } if pp.name != "" { tmpNameIndex.Add(pp.name, pp.id) hasNames = true @@ -659,13 +761,22 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { // Sync the CID index to what was build from pins. This fixes any invalid // indexes, which could happen if ipfs was terminated between writing pin // and writing secondary index. - changed, err := p.cidIndex.SyncTo(tmpCidIndex) + changed, err := p.cidRIndex.SyncTo(tmpCidRIndex) + if err != nil { + return fmt.Errorf("cannot sync indexes: %v", err) + } + if changed { + log.Error("invalid recursive indexes detected - rebuilt") + } + + changed, err = p.cidDIndex.SyncTo(tmpCidDIndex) if err != nil { - return fmt.Errorf("cannot sync cid indexes: %v", err) + return fmt.Errorf("cannot sync indexes: %v", err) } if changed { - log.Error("invalid cid indexes detected - rebuilt") + log.Error("invalid direct indexes detected - rebuilt") } + if hasNames { changed, err = p.nameIndex.SyncTo(tmpNameIndex) if err != nil { @@ -684,7 +795,25 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { p.lock.RLock() defer p.lock.RUnlock() - return p.directPin.Keys(), nil + var cids []cid.Cid + var e error + err := p.cidDIndex.ForEach("", func(idx, id string) bool { + var c cid.Cid + c, e = cid.Decode(idx) + if e != nil { + return false + } + cids = append(cids, c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cids, nil } // RecursiveKeys returns a slice containing the recursively pinned keys @@ -692,7 +821,25 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { p.lock.RLock() defer p.lock.RUnlock() - return p.recursePin.Keys(), nil + var cids []cid.Cid + var e error + err := p.cidRIndex.ForEach("", func(idx, id string) bool { + var c cid.Cid + c, e = cid.Decode(idx) + if e != nil { + return false + } + cids = append(cids, c) + return true + }) + if err != nil { + return nil, err + } + if e != nil { + return nil, e + } + + return cids, nil } // InternalPins returns all cids kept pinned for the internal state of the @@ -713,11 +860,15 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error p.lock.Lock() defer p.lock.Unlock() - if !p.recursePin.Has(from) { + has, err := p.cidRIndex.HasAny(from.String()) + if err != nil { + return err + } + if !has { return fmt.Errorf("'from' cid was not recursively pinned already") } - err := p.addPin(to, ipfspinner.Recursive, "") + err = p.addPin(to, ipfspinner.Recursive, "") if err != nil { return err } @@ -767,11 +918,11 @@ func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { // TODO: remove his to support multiple pins per CID switch mode { case ipfspinner.Recursive: - if p.recursePin.Has(c) { + if has, _ := p.cidRIndex.HasAny(c.String()); has { return // already a recursive pin for this CID } case ipfspinner.Direct: - if p.directPin.Has(c) { + if has, _ := p.cidDIndex.HasAny(c.String()); has { return // already a direct pin for this CID } default: diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 71697b4..0530323 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -14,6 +14,7 @@ import ( cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" + lds "github.com/ipfs/go-ds-leveldb" blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" ipfspin "github.com/ipfs/go-ipfs-pinner" @@ -494,8 +495,22 @@ func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) { } } +type batchWrap struct { + ds.Datastore +} + +func (d *batchWrap) Batch() (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} + func makeStore() (ds.Datastore, ipld.DAGService) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) + ldstore, err := lds.NewDatastore("", nil) + if err != nil { + panic(err) + } + var dstore ds.Batching + dstore = &batchWrap{ldstore} + bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) @@ -509,7 +524,7 @@ func BenchmarkLoadRebuild(b *testing.B) { dstore, dserv := makeStore() pinner := New(dstore, dserv) - nodes := makeNodes(1024, dserv) + nodes := makeNodes(4096, dserv) pinNodes(nodes, pinner, true) b.Run("RebuildTrue", func(b *testing.B) { @@ -535,25 +550,71 @@ func BenchmarkLoadRebuild(b *testing.B) { }) } -// BenchmarkPinSeries demonstrates creating individual pins. Each run in the +// BenchmarkNthPins shows the time it takes to create/save 1 pin when a number +// of other pins already exist. Each run in the series shows performance for +// creating a pin in a larger number of existing pins. +func BenchmarkNthPin(b *testing.B) { + dstore, dserv := makeStore() + pinner := New(dstore, dserv) + pinnerIPLD := ipldpinner.New(dstore, dserv, dserv) + + for count := 1000; count <= 10000; count += 1000 { + b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { + benchmarkNthPin(b, count, pinner, dserv) + }) + + b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { + benchmarkNthPin(b, count, pinnerIPLD, dserv) + }) + } +} + +func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { + ctx := context.Background() + nodes := makeNodes(count, dserv) + pinNodes(nodes[:count-1], pinner, true) + b.ResetTimer() + + which := count - 1 + for i := 0; i < b.N; i++ { + // Pin the Nth node and Flush + err := pinner.Pin(ctx, nodes[which], true) + if err != nil { + panic(err) + } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } + // Unpin the nodes so that it can pinned next iter. + b.StopTimer() + err = pinner.Unpin(ctx, nodes[which].Cid(), true) + if err != nil { + panic(err) + } + b.StartTimer() + } +} + +// BenchmarkNPins demonstrates creating individual pins. Each run in the // series shows performance for a larger number of individual pins. -func BenchmarkPinSeries(b *testing.B) { +func BenchmarkNPins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() pinner := New(dstore, dserv) - benchmarkPin(b, count, pinner, dserv) + benchmarkNPins(b, count, pinner, dserv) }) b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { dstore, dserv := makeStore() pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkPin(b, count, pinner, dserv) + benchmarkNPins(b, count, pinner, dserv) }) } } -func benchmarkPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { +func benchmarkNPins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { ctx := context.Background() nodes := makeNodes(count, dserv) b.ResetTimer() @@ -578,25 +639,25 @@ func benchmarkPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAG } } -// BenchmarkUnpinSeries demonstrates unpinning individual pins. Each run in the +// BenchmarkNUnpins demonstrates unpinning individual pins. Each run in the // series shows performance for a larger number of individual unpins. -func BenchmarkUnpinSeries(b *testing.B) { +func BenchmarkNUnpins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() pinner := New(dstore, dserv) - benchmarkPin(b, count, pinner, dserv) + benchmarkNUnpins(b, count, pinner, dserv) }) b.Run(fmt.Sprint("UninIPLD-", count), func(b *testing.B) { dstore, dserv := makeStore() pinner := ipldpinner.New(dstore, dserv, dserv) - benchmarkPin(b, count, pinner, dserv) + benchmarkNUnpins(b, count, pinner, dserv) }) } } -func benchmarkUnpin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { +func benchmarkNUnpins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { ctx := context.Background() nodes := makeNodes(count, dserv) pinNodes(nodes, pinner, true) @@ -623,7 +684,7 @@ func benchmarkUnpin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.D // BenchmarkPinAllSeries shows times to pin all nodes with only one Flush at // the end. -func BenchmarkPinAllSeries(b *testing.B) { +func BenchmarkPinAll(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { dstore, dserv := makeStore() diff --git a/go.mod b/go.mod index bd0869c..30a08c9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/ipfs/go-blockservice v0.1.2 github.com/ipfs/go-cid v0.0.3 - github.com/ipfs/go-datastore v0.3.0 + github.com/ipfs/go-datastore v0.4.5 + github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ipfs-blockstore v0.1.0 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-util v0.0.1 diff --git a/go.sum b/go.sum index c6a12e4..86e56c0 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -40,6 +41,7 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -76,12 +78,17 @@ github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.3.0 h1:9au0tYi/+n7xeUnGHG6davnS8x9hWbOzP/388Vx3CMs= -github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.1 h1:W4ZfzyhNi3xmuU5dQhjfuRn/wFuqEE1KnOmmQiOevEY= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= +github.com/ipfs/go-ds-leveldb v0.0.1 h1:Z0lsTFciec9qYsyngAw1f/czhRU35qBLR2vhavPFgqA= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9iaw= +github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= @@ -131,6 +138,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= @@ -143,6 +152,8 @@ github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muM github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -278,6 +289,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= @@ -306,24 +318,34 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -340,12 +362,19 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -353,3 +382,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 0244724dd288847f985e6e817bf6ac3a9274bb95 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 28 Oct 2020 23:44:04 -0700 Subject: [PATCH 09/29] Add comments and unit test --- dspinner/pin.go | 27 +++++++++-------- dspinner/pin_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++++ ipldpinner/pin.go | 2 +- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index b40edaa..4fc030d 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -1,5 +1,6 @@ -// Package pin implements structures and methods to keep track of -// which objects a user wants to keep stored locally. +// Package dspinner implements structures and methods to keep track of +// which objects a user wants to keep stored locally. This implementation +// stores pin data in a datastore. package dspinner import ( @@ -278,6 +279,8 @@ func (p *pinner) removePin(pp *pin) error { // Unpin a given key func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { + cidStr := c.String() + p.lock.Lock() defer p.lock.Unlock() @@ -288,7 +291,6 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } matches := p.Ls(matchSpec) */ - cidStr := c.String() has, err := p.cidRIndex.HasAny(cidStr) if err != nil { return err @@ -421,16 +423,17 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne return "", false, nil } -// CheckIfPinned Checks if a set of keys are pinned, more efficient than +// CheckIfPinned checks if a set of keys are pinned, more efficient than // calling IsPinned for each key, returns the pinned status of cid(s) // // TODO: If a CID is pinned by multiple pins, should they all be reported? func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() pinned := make([]ipfspinner.Pinned, 0, len(cids)) toCheck := cid.NewSet() + p.lock.RLock() + defer p.lock.RUnlock() + // First check for non-Indirect pins directly for _, c := range cids { cidStr := c.String() @@ -469,7 +472,7 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn toCheck.Remove(c) } - err := checkChildren(rk, c) + err = checkChildren(rk, c) if err != nil { return err } @@ -607,7 +610,7 @@ func (p *pinner) loadAllPins() ([]*pin, error) { pins := make([]*pin, len(ents)) for i := range ents { var p *pin - p, err := decodePin(path.Base(ents[i].Key), ents[i].Value) + p, err = decodePin(path.Base(ents[i].Key), ents[i].Value) if err != nil { return nil, err } @@ -860,11 +863,11 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error p.lock.Lock() defer p.lock.Unlock() - has, err := p.cidRIndex.HasAny(from.String()) + found, err := p.cidRIndex.HasAny(from.String()) if err != nil { return err } - if !has { + if !found { return fmt.Errorf("'from' cid was not recursively pinned already") } @@ -877,11 +880,11 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return nil } - ok, err := p.removePinsForCid(from, ipfspinner.Recursive) + found, err = p.removePinsForCid(from, ipfspinner.Recursive) if err != nil { return err } - if !ok { + if !found { log.Error("found CID index with missing pin") } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 0530323..d4e6745 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -376,6 +376,77 @@ func TestPinUpdate(t *testing.T) { assertPinned(t, p, c1, "c1 should be pinned now") } +func TestLoadDirty(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + + p := New(dstore, dserv) + + a, ak := randNode() + err := dserv.Add(ctx, a) + if err != nil { + t.Fatal(err) + } + + // Pin A{} recursive + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + + cidStr := ak.String() + + // Corrupt index + cidRIndex := p.(*pinner).cidRIndex + cidRIndex.DeleteAll(cidStr) + + // Verify dirty + data, err := dstore.Get(dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 1 { + t.Fatal("dirty flag not set") + } + + has, err := cidRIndex.HasAny(cidStr) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatal("index should be deleted") + } + + // Create new pinner on same datastore that was never flushed. + p, err = LoadPinner(dstore, dserv) + if err != nil { + t.Fatal(err) + } + + // Verify not dirty + data, err = dstore.Get(dirtyKey) + if err != nil { + t.Fatalf("could not read dirty flag: %v", err) + } + if data[0] != 0 { + t.Fatal("dirty flag is set") + } + + // Verify index rebuilt + cidRIndex = p.(*pinner).cidRIndex + has, err = cidRIndex.HasAny(cidStr) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatal("index should have been rebuilt") + } +} + func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { if aBranchLen < 3 { err = errors.New("set aBranchLen to at least 3") diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go index b2bda9d..742bd2b 100644 --- a/ipldpinner/pin.go +++ b/ipldpinner/pin.go @@ -1,6 +1,6 @@ // Package ipldpinner implements structures and methods to keep track of // which objects a user wants to keep stored locally. This implementation -// uses an IPLD DAG to determine indirect pins. +// stores pin information in a mdag structure. package ipldpinner import ( From 22a61da8c2e994270cd79adfe05643bda92a8944 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 29 Oct 2020 11:58:52 -0700 Subject: [PATCH 10/29] Speed up pinning by avoining 2nd recursive check if no changes Also, increased test coverage. --- dspinner/pin.go | 44 ++++++++++------ dspinner/pin_test.go | 118 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 16 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 4fc030d..eb5670e 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -78,7 +78,7 @@ type pinner struct { cidRIndex dsindex.Indexer nameIndex dsindex.Indexer - dirty bool + dirty uint64 } var _ ipfspinner.Pinner = (*pinner)(nil) @@ -144,23 +144,27 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return nil } + dirtyBefore := p.dirty + // temporary unlock to fetch the entire graph p.lock.Unlock() // Fetch graph to ensure that any children of the pinned node are in // the graph. err = mdag.FetchGraph(ctx, c, p.dserv) p.lock.Lock() - if err != nil { return err } - found, err = p.cidRIndex.HasAny(cidStr) - if err != nil { - return err - } - if found { - return nil + // Only look again if something has changed. + if p.dirty != dirtyBefore { + found, err = p.cidRIndex.HasAny(cidStr) + if err != nil { + return err + } + if found { + return nil + } } // TODO: remove this to support multiple pins per CID @@ -519,9 +523,6 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn // Use with care! If used improperly, garbage collection may not // be successful. func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { - p.lock.Lock() - defer p.lock.Unlock() - // Check cache to see if CID is pinned switch mode { case ipfspinner.Direct, ipfspinner.Recursive: @@ -530,6 +531,9 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { panic("unrecognized pin type") } + p.lock.Lock() + defer p.lock.Unlock() + p.removePinsForCid(c, mode) } @@ -720,7 +724,7 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, return nil, fmt.Errorf("cannot load dirty flag: %v", err) } if data[0] == 1 { - p.dirty = true + p.dirty = 1 pins, err := p.loadAllPins() if err != nil { @@ -1045,11 +1049,21 @@ func decodePin(pid string, data []byte) (*pin, error) { return p, nil } +// setDirty saves a boolean dirty flag in the datastore whenever there is a +// transition between a dirty (counter > 0) and non-dirty (counter == 0) state. func (p *pinner) setDirty(dirty bool) { - if p.dirty == dirty { - return + if dirty { + p.dirty++ + if p.dirty != 1 { + return // already > 1 + } + } else if p.dirty == 0 { + return // already 0 + } else { + p.dirty = 0 } - p.dirty = dirty + + // Do edge-triggered write to datastore data := []byte{0} if dirty { data[0] = 1 diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index d4e6745..39fa885 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -47,6 +47,30 @@ func assertPinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { } } +func assertPinnedWithType(t *testing.T, p ipfspin.Pinner, c cid.Cid, mode ipfspin.Mode, failmsg string) { + modeText, pinned, err := p.IsPinnedWithType(context.Background(), c, mode) + if err != nil { + t.Fatal(err) + } + + expect, ok := ipfspin.ModeToString(mode) + if !ok { + t.Fatal("unrecognized pin mode") + } + + if !pinned { + t.Fatal(failmsg) + } + + if mode == ipfspin.Any { + return + } + + if expect != modeText { + t.Fatal("expected", expect, "pin, got", modeText) + } +} + func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { _, pinned, err := p.IsPinned(context.Background(), c) if err != nil { @@ -82,6 +106,7 @@ func TestPinnerBasic(t *testing.T) { } assertPinned(t, p, ak, "Failed to find key") + assertPinnedWithType(t, p, ak, ipfspin.Direct, "Expected direct pin") // create new node c, to be indirectly pinned through b c, _ := randNode() @@ -117,7 +142,8 @@ func TestPinnerBasic(t *testing.T) { assertPinned(t, p, ck, "child of recursively pinned node not found") - assertPinned(t, p, bk, "Recursively pinned node not found..") + assertPinned(t, p, bk, "Pinned node not found") + assertPinnedWithType(t, p, bk, ipfspin.Recursive, "Recursively pinned node not found") d, _ := randNode() _ = d.AddNodeLink("a", a) @@ -145,6 +171,67 @@ func TestPinnerBasic(t *testing.T) { dk := d.Cid() assertPinned(t, p, dk, "pinned node not found.") + cids, err := p.RecursiveKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 2 { + t.Error("expected 2 recursive pins") + } + if !(bk == cids[0] || bk == cids[1]) { + t.Error("expected recursive pin of B") + } + if !(dk == cids[0] || dk == cids[1]) { + t.Error("expected recursive pin of D") + } + + pinned, err := p.CheckIfPinned(ctx, ak, bk, ck, dk) + if err != nil { + t.Fatal(err) + } + if len(pinned) != 4 { + t.Error("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + t.Error("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + t.Error("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + t.Error("C should be pinned indirectly") + } + if pn.Via != bk { + t.Error("C should be pinned via B") + } + case dk: + if pn.Mode != ipfspin.Recursive { + t.Error("D pinned with wrong mode") + } + } + } + + cids, err = p.DirectKeys(ctx) + if err != nil { + t.Fatal(err) + } + if len(cids) != 1 { + t.Error("expected 1 direct pin") + } + if cids[0] != ak { + t.Error("wrong direct pin") + } + + cids, _ = p.InternalPins(ctx) + if len(cids) != 0 { + t.Error("shound not have internal keys") + } + // Test recursive unpin err = p.Unpin(ctx, dk, true) if err != nil { @@ -190,6 +277,35 @@ func TestPinnerBasic(t *testing.T) { assertPinned(t, impPinner, bk, "could not find recursively pinned node") } +func TestRemovePinWithMode(t *testing.T) { + ctx := context.Background() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + p := New(dstore, dserv) + + a, ak := randNode() + dserv.Add(ctx, a) + + p.Pin(ctx, a, false) + + ok, err := p.(*pinner).removePinsForCid(ak, ipfspin.Recursive) + if err != nil { + t.Fatal(err) + } + if ok { + t.Error("pin should not have been removed") + } + + p.RemovePinWithMode(ak, ipfspin.Direct) + + assertUnpinned(t, p, ak, "pin was not removed") +} + func TestIsPinnedLookup(t *testing.T) { // Test that lookups work in pins which share // the same branches. For that construct this tree: From 1662bb8c06934e287d683fc17b7a3f518dce08da Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 29 Oct 2020 12:07:12 -0700 Subject: [PATCH 11/29] correct log level --- dspinner/pin.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index eb5670e..ec0cf79 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -773,7 +773,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { return fmt.Errorf("cannot sync indexes: %v", err) } if changed { - log.Error("invalid recursive indexes detected - rebuilt") + log.Info("invalid recursive indexes detected - rebuilt") } changed, err = p.cidDIndex.SyncTo(tmpCidDIndex) @@ -781,7 +781,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { return fmt.Errorf("cannot sync indexes: %v", err) } if changed { - log.Error("invalid direct indexes detected - rebuilt") + log.Info("invalid direct indexes detected - rebuilt") } if hasNames { @@ -790,7 +790,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { return fmt.Errorf("cannot sync name indexes: %v", err) } if changed { - log.Error("invalid name indexes detected - rebuilt") + log.Info("invalid name indexes detected - rebuilt") } } From cad837873ee6c3b7684a584fa979011a181b2687 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 29 Oct 2020 16:34:47 -0700 Subject: [PATCH 12/29] improve import/export unit test --- dspinner/pin_test.go | 99 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 14 deletions(-) diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 39fa885..2cdfe4f 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -122,7 +122,6 @@ func TestPinnerBasic(t *testing.T) { if err != nil { t.Fatal(err) } - err = b.AddNodeLink("otherchild", c) if err != nil { t.Fatal(err) @@ -146,11 +145,11 @@ func TestPinnerBasic(t *testing.T) { assertPinnedWithType(t, p, bk, ipfspin.Recursive, "Recursively pinned node not found") d, _ := randNode() - _ = d.AddNodeLink("a", a) - _ = d.AddNodeLink("c", c) + d.AddNodeLink("a", a) + d.AddNodeLink("c", c) e, _ := randNode() - _ = d.AddNodeLink("e", e) + d.AddNodeLink("e", e) // Must be in dagserv for unpin to work err = dserv.Add(ctx, e) @@ -206,8 +205,8 @@ func TestPinnerBasic(t *testing.T) { if pn.Mode != ipfspin.Indirect { t.Error("C should be pinned indirectly") } - if pn.Via != bk { - t.Error("C should be pinned via B") + if pn.Via != dk && pn.Via != bk { + t.Error("C should be pinned via D or B") } case dk: if pn.Mode != ipfspin.Recursive { @@ -243,16 +242,84 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv) + p, err = LoadPinner(dstore, dserv) if err != nil { t.Fatal(err) } // Test directly pinned - assertPinned(t, np, ak, "Could not find pinned node!") + assertPinned(t, p, ak, "Could not find pinned node!") // Test recursively pinned - assertPinned(t, np, bk, "could not find recursively pinned node") + assertPinned(t, p, bk, "could not find recursively pinned node") +} + +func TestImportExport(t *testing.T) { + ctx := context.Background() + dstore, dserv := makeStore() + p := New(dstore, dserv) + + a, ak := randNode() + err := p.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + // create new node c, to be indirectly pinned through b + c, ck := randNode() + dserv.Add(ctx, c) + + // Create new node b, to be parent to a and c + b, _ := randNode() + b.AddNodeLink("child", a) + b.AddNodeLink("otherchild", c) + bk := b.Cid() // CID changed after adding links + + // recursively pin B{A,C} + err = p.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + err = p.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + verifyPins := func(pinner ipfspin.Pinner) error { + pinned, err := pinner.CheckIfPinned(ctx, ak, bk, ck) + if err != nil { + return err + } + if len(pinned) != 3 { + return errors.New("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + return errors.New("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + return errors.New("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + return errors.New("C should be pinned indirectly") + } + if pn.Via != bk { + return errors.New("C should be pinned via B") + } + } + } + return nil + } + + err = verifyPins(p) + if err != nil { + t.Fatal(err) + } ipldPinner, expCount, err := ExportToIPLDPinner(dstore, dserv, dserv) if err != nil { @@ -262,10 +329,12 @@ func TestPinnerBasic(t *testing.T) { t.Fatal("expected 2 exported pins, got", expCount) } - assertPinned(t, ipldPinner, ak, "Could not find pinned node!") - assertPinned(t, ipldPinner, bk, "could not find recursively pinned node") + err = verifyPins(ipldPinner) + if err != nil { + t.Fatal(err) + } - impPinner, impCount, err := ImportFromIPLDPinner(dstore, dserv, dserv) + importPinner, impCount, err := ImportFromIPLDPinner(dstore, dserv, dserv) if err != nil { t.Fatal(err) } @@ -273,8 +342,10 @@ func TestPinnerBasic(t *testing.T) { t.Fatal("expected", expCount, "imported pins, got", impCount) } - assertPinned(t, impPinner, ak, "Could not find pinned node!") - assertPinned(t, impPinner, bk, "could not find recursively pinned node") + err = verifyPins(importPinner) + if err != nil { + t.Fatal(err) + } } func TestRemovePinWithMode(t *testing.T) { From d1a44d7c1f6701a020a1a705012d825bce358cfa Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 16 Nov 2020 13:37:49 -0800 Subject: [PATCH 13/29] Update returns error if from CID is not pinned, even when from and to are same This addresses issue https://github.com/ipfs/go-ipfs/issues/7745 --- dspinner/pin.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index ec0cf79..9ab2086 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -860,10 +860,6 @@ func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) { // // TODO: This will not work when multiple pins are supported func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error { - if from == to { - return nil - } - p.lock.Lock() defer p.lock.Unlock() @@ -875,6 +871,11 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return fmt.Errorf("'from' cid was not recursively pinned already") } + // If `from` already recusively pinned and `to` is the same, then all done + if from == to { + return nil + } + err = p.addPin(to, ipfspinner.Recursive, "") if err != nil { return err From 9bb7a0c0382cc7707a1e07c7e63f874802b47447 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 16 Nov 2020 13:53:30 -0800 Subject: [PATCH 14/29] test update of same pin --- dspinner/pin_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 2cdfe4f..6085905 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -535,6 +535,7 @@ func TestPinUpdate(t *testing.T) { p := New(dstore, dserv) n1, c1 := randNode() n2, c2 := randNode() + _, c3 := randNode() err := dserv.Add(ctx, n1) if err != nil { @@ -559,6 +560,31 @@ func TestPinUpdate(t *testing.T) { t.Fatal(err) } + // Test updating same pin that is already pinned. + if err = p.Update(ctx, c2, c2, true); err != nil { + t.Fatal(err) + } + // Check that pin is still pinned. + _, ok, err := p.IsPinned(ctx, c2) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("c2 should still be pinned") + } + + // Test updating same pin that is not pinned. + if err = p.Update(ctx, c3, c3, false); err == nil { + t.Fatal("expected error updating unpinned cid") + } + _, ok, err = p.IsPinned(ctx, c3) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("c3 should not be pinned") + } + assertPinned(t, p, c2, "c2 should be pinned still") assertPinned(t, p, c1, "c1 should be pinned now") } From eb32271dfc514328c31b0705e0c92c33dc113660 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 16 Nov 2020 17:30:32 -0800 Subject: [PATCH 15/29] Cleanup and better test coverage --- dspinner/pin.go | 45 +++++++++++++++------------------- dspinner/pin_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 9ab2086..feb3a05 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -36,7 +36,7 @@ var ( // ErrNotPinned is returned when trying to unpin items that are not pinned. ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") - log = logging.Logger("pin") + log logging.StandardLogger = logging.Logger("pin") linkDirect, linkRecursive string @@ -91,8 +91,6 @@ type pin struct { name string } -func (p *pin) codec() uint64 { return p.cid.Type() } -func (p *pin) version() uint64 { return p.cid.Version() } func (p *pin) dsKey() ds.Key { return ds.NewKey(path.Join(pinKeyPath, p.id)) } @@ -173,12 +171,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return err } if found { - ok, _ := p.removePinsForCid(c, ipfspinner.Direct) - if !ok { - // Fix index; remove index for pin that does not exist - p.cidDIndex.DeleteAll(c.String()) - log.Error("found cid index for direct pin that does not exist") - } + p.removePinsForCid(c, ipfspinner.Direct) } err = p.addPin(c, ipfspinner.Recursive, "") @@ -299,7 +292,6 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { if err != nil { return err } - var ok bool if has { if !recursive { @@ -315,16 +307,10 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } } - ok, err = p.removePinsForCid(c, ipfspinner.Any) + _, err = p.removePinsForCid(c, ipfspinner.Any) if err != nil { return err } - if !ok { - p.setDirty(true) - p.cidRIndex.DeleteAll(cidStr) - p.cidDIndex.DeleteAll(cidStr) - log.Error("found CID index with missing pin") - } return nil } @@ -537,18 +523,18 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { p.removePinsForCid(c, mode) } -// removePinsForCid removes all pins for a cid that have the specified mode. +// removePinsForCid removes all pins for a cid that has the specified mode. func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) { // Search for pins by CID var ids []string var err error + cidStr := c.String() switch mode { case ipfspinner.Recursive: - ids, err = p.cidRIndex.Search(c.String()) + ids, err = p.cidRIndex.Search(cidStr) case ipfspinner.Direct: - ids, err = p.cidDIndex.Search(c.String()) + ids, err = p.cidDIndex.Search(cidStr) case ipfspinner.Any: - cidStr := c.String() ids, err = p.cidRIndex.Search(cidStr) dIds, dErr := p.cidDIndex.Search(cidStr) if dErr != nil && err == nil { @@ -570,6 +556,18 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) pp, err = p.loadPin(pid) if err != nil { if err == ds.ErrNotFound { + p.setDirty(true) + // Fix index; remove index for pin that does not exist + switch mode { + case ipfspinner.Recursive: + p.cidRIndex.DeleteAll(cidStr) + case ipfspinner.Direct: + p.cidDIndex.DeleteAll(cidStr) + case ipfspinner.Any: + p.cidRIndex.DeleteAll(cidStr) + p.cidDIndex.DeleteAll(cidStr) + } + log.Error("found CID index with missing pin") continue } return false, err @@ -885,13 +883,10 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return nil } - found, err = p.removePinsForCid(from, ipfspinner.Recursive) + _, err = p.removePinsForCid(from, ipfspinner.Recursive) if err != nil { return err } - if !found { - log.Error("found CID index with missing pin") - } return nil } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 6085905..0f7c2c5 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -21,10 +21,24 @@ import ( "github.com/ipfs/go-ipfs-pinner/ipldpinner" util "github.com/ipfs/go-ipfs-util" ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" ) var rand = util.NewTimeSeededRand() +type fakeLogger struct { + logging.StandardLogger + lastError error +} + +func (f *fakeLogger) Error(args ...interface{}) { + f.lastError = errors.New(fmt.Sprint(args...)) +} + +func (f *fakeLogger) Errorf(format string, args ...interface{}) { + f.lastError = fmt.Errorf(format, args...) +} + func randNode() (*mdag.ProtoNode, cid.Cid) { nd := new(mdag.ProtoNode) nd.SetData(make([]byte, 32)) @@ -231,12 +245,22 @@ func TestPinnerBasic(t *testing.T) { t.Error("shound not have internal keys") } + err = p.Unpin(ctx, dk, false) + if err == nil { + t.Fatal("expected error unpinning recursive pin without specifying recursive") + } + // Test recursive unpin err = p.Unpin(ctx, dk, true) if err != nil { t.Fatal(err) } + err = p.Unpin(ctx, dk, true) + if err != ErrNotPinned { + t.Fatal("expected error:", ErrNotPinned) + } + err = p.Flush(ctx) if err != nil { t.Fatal(err) @@ -252,6 +276,40 @@ func TestPinnerBasic(t *testing.T) { // Test recursively pinned assertPinned(t, p, bk, "could not find recursively pinned node") + + // Remove the pin but not the index to simulate corruption + dsp := p.(*pinner) + ids, err := dsp.cidDIndex.Search(ak.String()) + if err != nil { + t.Fatal(err) + } + if len(ids) == 0 { + t.Fatal("did not find pin for cid", ak.String()) + } + pp, err := dsp.loadPin(ids[0]) + if err != nil { + t.Fatal(err) + } + err = dsp.dstore.Delete(pp.dsKey()) + if err != nil { + t.Fatal(err) + } + + realLog := log + fakeLog := &fakeLogger{} + fakeLog.StandardLogger = log + log = fakeLog + err = p.Pin(ctx, a, true) + if err != nil { + t.Fatal(err) + } + if fakeLog.lastError == nil { + t.Error("expected error to be logged") + } else if fakeLog.lastError.Error() != "found CID index with missing pin" { + t.Error("did not get expected log message") + } + + log = realLog } func TestImportExport(t *testing.T) { From 22254b2614819cff6781830327fa48b784dfd3ef Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 00:26:48 -0800 Subject: [PATCH 16/29] Change requested in review This includes moving the pin converstion logic into the pinconv package. --- dsindex/error.go | 8 ++ dsindex/indexer.go | 74 ++++++++++++---- dsindex/indexer_test.go | 34 ++++++-- dspinner/pin.go | 183 +++++++++++++--------------------------- dspinner/pin_test.go | 130 ++++++---------------------- pinconv/pinconv.go | 128 ++++++++++++++++++++++++++++ pinconv/pinconv_test.go | 153 +++++++++++++++++++++++++++++++++ 7 files changed, 451 insertions(+), 259 deletions(-) create mode 100644 dsindex/error.go create mode 100644 pinconv/pinconv.go create mode 100644 pinconv/pinconv_test.go diff --git a/dsindex/error.go b/dsindex/error.go new file mode 100644 index 0000000..b61d90d --- /dev/null +++ b/dsindex/error.go @@ -0,0 +1,8 @@ +package dsindex + +import "errors" + +var ( + ErrEmptyIndex = errors.New("index is empty") + ErrEmptyKey = errors.New("key is empty") +) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index f084d45..ead822b 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -11,23 +11,26 @@ import ( // Indexer maintains a secondary index. Each value of the secondary index maps // to one more primary keys. type Indexer interface { - // Add adds the a to the an index + // Add adds the specified key to the an index Add(index, id string) error // Delete deletes the specified key from the index. If the key is not in // the datastore, this method returns no error. Delete(index, id string) error - // DeleteAll deletes all keys in the given index. If a key is not in the - // datastore, this method returns no error. - DeleteAll(index string) (count int, err error) + // DeleteIndex deletes all keys in the given index. If an index is not in + // the datastore, this method returns no error. + DeleteIndex(index string) (count int, err error) + + // DeleteAll deletes all indexes managed by this Indexer + DeleteAll() (count int, err error) // ForEach calls the function for each key in the specified index, until // there are no more keys, or until the function returns false. If index - // is empty string, then all index names are iterated. + // is empty string, then all indexs are iterated. ForEach(index string, fn func(index, id string) bool) error - // HasKey determines the specified index contains the specified primary key + // HasKey determines if the index contains the specified key HasKey(index, id string) (bool, error) // HasAny determines if any key is in the specified index. If index is @@ -54,6 +57,9 @@ type indexer struct { // New creates a new datastore index. All indexes are stored prefixed with the // specified index path. +// +// To persist the actions of calling Indexer functions, it is necessary to call +// dstore.Sync. func New(dstore ds.Datastore, indexPath string) Indexer { return &indexer{ dstore: dstore, @@ -62,28 +68,35 @@ func New(dstore ds.Datastore, indexPath string) Indexer { } func (x *indexer) Add(index, id string) error { + if index == "" { + return ErrEmptyIndex + } + if id == "" { + return ErrEmptyKey + } key := ds.NewKey(path.Join(x.indexPath, index, id)) return x.dstore.Put(key, []byte{}) } func (x *indexer) Delete(index, id string) error { + if index == "" { + return ErrEmptyIndex + } + if id == "" { + return ErrEmptyKey + } return x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, index, id))) } -func (x *indexer) DeleteAll(index string) (int, error) { - ents, err := x.queryPrefix(path.Join(x.indexPath, index)) - if err != nil { - return 0, err - } - - for i := range ents { - err = x.dstore.Delete(ds.NewKey(ents[i].Key)) - if err != nil { - return 0, err - } +func (x *indexer) DeleteIndex(index string) (int, error) { + if index == "" { + return 0, ErrEmptyIndex } + return x.deletePrefix(path.Join(x.indexPath, index)) +} - return len(ents), nil +func (x *indexer) DeleteAll() (int, error) { + return x.deletePrefix(x.indexPath) } func (x *indexer) ForEach(index string, fn func(idx, id string) bool) error { @@ -117,6 +130,12 @@ func (x *indexer) ForEach(index string, fn func(idx, id string) bool) error { } func (x *indexer) HasKey(index, id string) (bool, error) { + if index == "" { + return false, ErrEmptyIndex + } + if id == "" { + return false, ErrEmptyKey + } return x.dstore.Has(ds.NewKey(path.Join(x.indexPath, index, id))) } @@ -130,6 +149,9 @@ func (x *indexer) HasAny(index string) (bool, error) { } func (x *indexer) Search(index string) ([]string, error) { + if index == "" { + return nil, ErrEmptyIndex + } ents, err := x.queryPrefix(path.Join(x.indexPath, index)) if err != nil { return nil, err @@ -194,6 +216,22 @@ func (x *indexer) SyncTo(ref Indexer) (bool, error) { return len(refs) != 0 || len(delKeys) != 0, nil } +func (x *indexer) deletePrefix(prefix string) (int, error) { + ents, err := x.queryPrefix(prefix) + if err != nil { + return 0, err + } + + for i := range ents { + err = x.dstore.Delete(ds.NewKey(ents[i].Key)) + if err != nil { + return 0, err + } + } + + return len(ents), nil +} + func (x *indexer) queryPrefix(prefix string) ([]query.Entry, error) { q := query.Query{ Prefix: prefix, diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go index c8b679b..72783d9 100644 --- a/dsindex/indexer_test.go +++ b/dsindex/indexer_test.go @@ -28,9 +28,15 @@ func TestAdd(t *testing.T) { if err != nil { t.Fatal(err) } - err = nameIndex.Add("", "s1") - if err != nil { - t.Fatal(err) + + err = nameIndex.Add("", "noindex") + if err != ErrEmptyIndex { + t.Fatal("unexpected error:", err) + } + + err = nameIndex.Add("nokey", "") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) } } @@ -52,6 +58,16 @@ func TestHasKey(t *testing.T) { if ok { t.Fatal("should not have index") } + + _, err = nameIndex.HasKey("", "b1") + if err != ErrEmptyIndex { + t.Fatal("unexpected error:", err) + } + + _, err = nameIndex.HasKey("bob", "") + if err != ErrEmptyKey { + t.Fatal("unexpected error:", err) + } } func TestHasAny(t *testing.T) { @@ -71,11 +87,11 @@ func TestHasAny(t *testing.T) { t.Fatal(err) } if !ok { - t.Fatal("missing indexe", idx) + t.Fatal("missing index", idx) } } - count, err := nameIndex.DeleteAll("") + count, err := nameIndex.DeleteAll() if err != nil { t.Fatal(err) } @@ -153,7 +169,7 @@ func TestSearch(t *testing.T) { } } if ids[0] == ids[1] { - t.Fatal("dublicate id") + t.Fatal("duplicate id") } ids, err = nameIndex.Search("cathy") @@ -191,10 +207,10 @@ func TestDelete(t *testing.T) { t.Fatal(err) } if ok { - t.Fatal("index should have been deleted") + t.Fatal("index key should have been deleted") } - count, err := nameIndex.DeleteAll("bob") + count, err := nameIndex.DeleteIndex("bob") if err != nil { t.Fatal(err) } @@ -207,7 +223,7 @@ func TestDelete(t *testing.T) { } } -func TestSyndTo(t *testing.T) { +func TestSyncTo(t *testing.T) { nameIndex := createIndexer() dstore := ds.NewMapDatastore() diff --git a/dspinner/pin.go b/dspinner/pin.go index feb3a05..986768b 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -16,20 +16,20 @@ import ( "github.com/ipfs/go-datastore/query" ipfspinner "github.com/ipfs/go-ipfs-pinner" "github.com/ipfs/go-ipfs-pinner/dsindex" - "github.com/ipfs/go-ipfs-pinner/ipldpinner" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" mdag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-merkledag/dagutils" "github.com/polydawn/refmt/cbor" ) const ( - loadTimeout = 5 * time.Second - rebuildTimeout = 2 * time.Minute + loadTimeout = 5 * time.Second - pinKeyPath = "/.pins/pin" - indexKeyPath = "/.pins/index" - dirtyKeyPath = "/.pins/state/dirty" + basePath = "/pins" + pinKeyPath = "/pins/pin" + indexKeyPath = "/pins/index" + dirtyKeyPath = "/pins/state/dirty" ) var ( @@ -44,8 +44,6 @@ var ( pinCidRIndexPath string pinNameIndexPath string - pinDatastoreKey = ds.NewKey("/local/pins") - dirtyKey = ds.NewKey(dirtyKeyPath) ) @@ -78,7 +76,8 @@ type pinner struct { cidRIndex dsindex.Indexer nameIndex dsindex.Indexer - dirty uint64 + clean int + dirty int } var _ ipfspinner.Pinner = (*pinner)(nil) @@ -146,8 +145,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { // temporary unlock to fetch the entire graph p.lock.Unlock() - // Fetch graph to ensure that any children of the pinned node are in - // the graph. + // Fetch graph starting a node identified by cid err = mdag.FetchGraph(ctx, c, p.dserv) p.lock.Lock() if err != nil { @@ -228,7 +226,7 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { } } - // Store the pin + // Store the pin. Pin must be stored after index for recovery to work. err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { if mode == ipfspinner.Recursive { @@ -248,7 +246,8 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { func (p *pinner) removePin(pp *pin) error { p.setDirty(true) - // Remove pin from datastore + // Remove pin from datastore. Pin must be removed before index for + // recovery to work. err := p.dstore.Delete(pp.dsKey()) if err != nil { return err @@ -524,6 +523,8 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { } // removePinsForCid removes all pins for a cid that has the specified mode. +// Returns true if any pins, and all corresponding CID index entries, were +// removed. Otherwise, returns false. func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) { // Search for pins by CID var ids []string @@ -536,9 +537,12 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) ids, err = p.cidDIndex.Search(cidStr) case ipfspinner.Any: ids, err = p.cidRIndex.Search(cidStr) - dIds, dErr := p.cidDIndex.Search(cidStr) - if dErr != nil && err == nil { - err = dErr + if err != nil { + return false, err + } + dIds, err := p.cidDIndex.Search(cidStr) + if err != nil { + return false, err } if len(dIds) != 0 { ids = append(ids, dIds...) @@ -560,12 +564,12 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) // Fix index; remove index for pin that does not exist switch mode { case ipfspinner.Recursive: - p.cidRIndex.DeleteAll(cidStr) + p.cidRIndex.DeleteIndex(cidStr) case ipfspinner.Direct: - p.cidDIndex.DeleteAll(cidStr) + p.cidDIndex.DeleteIndex(cidStr) case ipfspinner.Any: - p.cidRIndex.DeleteAll(cidStr) - p.cidDIndex.DeleteAll(cidStr) + p.cidRIndex.DeleteIndex(cidStr) + p.cidDIndex.DeleteIndex(cidStr) } log.Error("found CID index with missing pin") continue @@ -621,97 +625,8 @@ func (p *pinner) loadAllPins() ([]*pin, error) { return pins, nil } -// ImportFromIPLDPinner converts pins stored in mdag based storage to pins -// stores in the datastore. Returns a dspinner loaded with the exported pins, -// and a count of the pins imported. -// -// After pins are stored in datastore, the root pin key is deleted to unlink -// the pin data in the DAGService. -func ImportFromIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { - ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) - if err != nil { - return nil, 0, err - } - - p := New(dstore, dserv).(*pinner) - - ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) - defer cancel() - - // Save pinned CIDs as new pins in datastore. - rCids, _ := ipldPinner.RecursiveKeys(ctx) - for i := range rCids { - err = p.addPin(rCids[i], ipfspinner.Recursive, "") - if err != nil { - return nil, 0, err - } - } - dCids, _ := ipldPinner.DirectKeys(ctx) - for i := range dCids { - err = p.addPin(dCids[i], ipfspinner.Direct, "") - if err != nil { - return nil, 0, err - } - } - - // Delete root mdag key from datastore to remove old pin storage. - if err = dstore.Delete(pinDatastoreKey); err != nil { - return nil, 0, fmt.Errorf("cannot delete old pin state: %v", err) - } - if err = dstore.Sync(pinDatastoreKey); err != nil { - return nil, 0, fmt.Errorf("cannot sync old pin state: %v", err) - } - - return p, len(rCids) + len(dCids), nil -} - -// ExportToIPLDPinner exports the pins stored in the datastore by dspinner, and -// imports them into the given internal DAGService. Returns an ipldpinner -// loaded with the exported pins, and a count of the pins exported. -// -// After the pins are stored in the DAGService, the pins and their indexes are -// removed. -func ExportToIPLDPinner(dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { - p := New(dstore, dserv).(*pinner) - pins, err := p.loadAllPins() - if err != nil { - return nil, 0, fmt.Errorf("cannot load pins: %v", err) - } - - ipldPinner := ipldpinner.New(dstore, dserv, internal) - - seen := cid.NewSet() - for _, pp := range pins { - if seen.Has(pp.cid) { - // multiple pins not support; can only keep one - continue - } - seen.Add(pp.cid) - ipldPinner.PinWithMode(pp.cid, pp.mode) - } - - ctx := context.TODO() - - // Save the ipldpinner pins - err = ipldPinner.Flush(ctx) - if err != nil { - return nil, 0, err - } - - // Remove the dspinner pins and indexes - for _, pp := range pins { - p.removePin(pp) - } - err = p.Flush(ctx) - if err != nil { - return nil, 0, err - } - - return ipldPinner, seen.Len(), nil -} - // LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { +func LoadPinner(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { p := New(dstore, dserv).(*pinner) data, err := dstore.Get(dirtyKey) @@ -729,9 +644,6 @@ func LoadPinner(dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, return nil, fmt.Errorf("cannot load pins: %v", err) } - ctx, cancel := context.WithTimeout(context.TODO(), rebuildTimeout) - defer cancel() - err = p.rebuildIndexes(ctx, pins) if err != nil { return nil, fmt.Errorf("cannot rebuild indexes: %v", err) @@ -800,7 +712,7 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { p.lock.RLock() defer p.lock.RUnlock() - var cids []cid.Cid + cidSet := cid.NewSet() var e error err := p.cidDIndex.ForEach("", func(idx, id string) bool { var c cid.Cid @@ -808,7 +720,7 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { if e != nil { return false } - cids = append(cids, c) + cidSet.Add(c) return true }) if err != nil { @@ -818,7 +730,7 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { return nil, e } - return cids, nil + return cidSet.Keys(), nil } // RecursiveKeys returns a slice containing the recursively pinned keys @@ -826,7 +738,7 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { p.lock.RLock() defer p.lock.RUnlock() - var cids []cid.Cid + cidSet := cid.NewSet() var e error err := p.cidRIndex.ForEach("", func(idx, id string) bool { var c cid.Cid @@ -834,7 +746,7 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { if e != nil { return false } - cids = append(cids, c) + cidSet.Add(c) return true }) if err != nil { @@ -844,7 +756,7 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { return nil, e } - return cids, nil + return cidSet.Keys(), nil } // InternalPins returns all cids kept pinned for the internal state of the @@ -869,11 +781,29 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return fmt.Errorf("'from' cid was not recursively pinned already") } - // If `from` already recusively pinned and `to` is the same, then all done + // If `from` already recursively pinned and `to` is the same, then all done if from == to { return nil } + // Check if the `to` cid is already recursively pinned + found, err = p.cidRIndex.HasAny(to.String()) + if err != nil { + return err + } + if found { + return fmt.Errorf("'to' cid was already recursively pinned") + } + + // Temporarily unlock while we fetch the differences. + p.lock.Unlock() + err = dagutils.DiffEnumerate(ctx, p.dserv, from, to) + p.lock.Lock() + + if err != nil { + return err + } + err = p.addPin(to, ipfspinner.Recursive, "") if err != nil { return err @@ -903,7 +833,7 @@ func (p *pinner) Flush(ctx context.Context) error { } // Sync pins and indexes - if err := p.dstore.Sync(ds.NewKey(pinKeyPath)); err != nil { + if err := p.dstore.Sync(ds.NewKey(basePath)); err != nil { return fmt.Errorf("cannot sync pin state: %v", err) } @@ -1048,15 +978,16 @@ func decodePin(pid string, data []byte) (*pin, error) { // setDirty saves a boolean dirty flag in the datastore whenever there is a // transition between a dirty (counter > 0) and non-dirty (counter == 0) state. func (p *pinner) setDirty(dirty bool) { + isClean := p.dirty == p.clean if dirty { p.dirty++ - if p.dirty != 1 { - return // already > 1 + if !isClean { + return // do not save; was already dirty } - } else if p.dirty == 0 { - return // already 0 + } else if isClean { + return // already clean } else { - p.dirty = 0 + p.clean = p.dirty // set clean } // Do edge-triggered write to datastore diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 0f7c2c5..ee0ac44 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -266,7 +266,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - p, err = LoadPinner(dstore, dserv) + p, err = LoadPinner(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -312,100 +312,6 @@ func TestPinnerBasic(t *testing.T) { log = realLog } -func TestImportExport(t *testing.T) { - ctx := context.Background() - dstore, dserv := makeStore() - p := New(dstore, dserv) - - a, ak := randNode() - err := p.Pin(ctx, a, false) - if err != nil { - t.Fatal(err) - } - - // create new node c, to be indirectly pinned through b - c, ck := randNode() - dserv.Add(ctx, c) - - // Create new node b, to be parent to a and c - b, _ := randNode() - b.AddNodeLink("child", a) - b.AddNodeLink("otherchild", c) - bk := b.Cid() // CID changed after adding links - - // recursively pin B{A,C} - err = p.Pin(ctx, b, true) - if err != nil { - t.Fatal(err) - } - - err = p.Flush(ctx) - if err != nil { - t.Fatal(err) - } - - verifyPins := func(pinner ipfspin.Pinner) error { - pinned, err := pinner.CheckIfPinned(ctx, ak, bk, ck) - if err != nil { - return err - } - if len(pinned) != 3 { - return errors.New("incorrect number of results") - } - for _, pn := range pinned { - switch pn.Key { - case ak: - if pn.Mode != ipfspin.Direct { - return errors.New("A pinned with wrong mode") - } - case bk: - if pn.Mode != ipfspin.Recursive { - return errors.New("B pinned with wrong mode") - } - case ck: - if pn.Mode != ipfspin.Indirect { - return errors.New("C should be pinned indirectly") - } - if pn.Via != bk { - return errors.New("C should be pinned via B") - } - } - } - return nil - } - - err = verifyPins(p) - if err != nil { - t.Fatal(err) - } - - ipldPinner, expCount, err := ExportToIPLDPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } - if expCount != 2 { - t.Fatal("expected 2 exported pins, got", expCount) - } - - err = verifyPins(ipldPinner) - if err != nil { - t.Fatal(err) - } - - importPinner, impCount, err := ImportFromIPLDPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } - if impCount != expCount { - t.Fatal("expected", expCount, "imported pins, got", impCount) - } - - err = verifyPins(importPinner) - if err != nil { - t.Fatal(err) - } -} - func TestRemovePinWithMode(t *testing.T) { ctx := context.Background() @@ -456,8 +362,12 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) - // TODO does pinner need to share datastore with blockservice? - p := New(dstore, dserv) + // Create new pinner. LoadPinner will not load anything since there are + // no pins saved in the datastore yet. + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } aKeys, bk, ck, err := makeTree(ctx, aBranchLen, dserv, p) if err != nil { @@ -494,11 +404,13 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) - // TODO does pinner need to share datastore with blockservice? - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -523,16 +435,20 @@ func TestDuplicateSemantics(t *testing.T) { } func TestFlush(t *testing.T) { + ctx := context.Background() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } _, k := randNode() p.PinWithMode(k, ipfspin.Recursive) - if err := p.Flush(context.Background()); err != nil { + if err = p.Flush(ctx); err != nil { t.Fatal(err) } assertPinned(t, p, k, "expected key to still be pinned") @@ -673,7 +589,7 @@ func TestLoadDirty(t *testing.T) { // Corrupt index cidRIndex := p.(*pinner).cidRIndex - cidRIndex.DeleteAll(cidStr) + cidRIndex.DeleteIndex(cidStr) // Verify dirty data, err := dstore.Get(dirtyKey) @@ -693,7 +609,7 @@ func TestLoadDirty(t *testing.T) { } // Create new pinner on same datastore that was never flushed. - p, err = LoadPinner(dstore, dserv) + p, err = LoadPinner(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -869,11 +785,13 @@ func BenchmarkLoadRebuild(b *testing.B) { nodes := makeNodes(4096, dserv) pinNodes(nodes, pinner, true) + ctx := context.Background() + b.Run("RebuildTrue", func(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{1}) - _, err := LoadPinner(dstore, dserv) + _, err := LoadPinner(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -884,7 +802,7 @@ func BenchmarkLoadRebuild(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{0}) - _, err := LoadPinner(dstore, dserv) + _, err := LoadPinner(ctx, dstore, dserv) if err != nil { panic(err.Error()) } diff --git a/pinconv/pinconv.go b/pinconv/pinconv.go new file mode 100644 index 0000000..0c1c093 --- /dev/null +++ b/pinconv/pinconv.go @@ -0,0 +1,128 @@ +// Package pinconv converts pins between the dag-based ipldpinner and the +// datastore-based dspinner. Once conversion is complete, the pins from the +// source pinner are removed. +package pinconv + +import ( + "context" + "fmt" + + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ipfspinner "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dspinner" + "github.com/ipfs/go-ipfs-pinner/ipldpinner" + ipld "github.com/ipfs/go-ipld-format" +) + +// ConvertPinsFromIPLDToDS converts pins stored in mdag based storage to pins +// stores in the datastore. Returns a dspinner loaded with the converted pins, +// and a count of the recursive and direct pins converted. +// +// After pins are stored in datastore, the root pin key is deleted to unlink +// the pin data in the DAGService. +func ConvertPinsFromIPLDToDS(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { + const ipldPinPath = "/local/pins" + + ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) + if err != nil { + return nil, 0, err + } + + dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + if err != nil { + return nil, 0, err + } + + seen := cid.NewSet() + cids, err := ipldPinner.RecursiveKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + seen.Add(cids[i]) + dsPinner.PinWithMode(cids[i], ipfspinner.Recursive) + } + convCount := len(cids) + + cids, err = ipldPinner.DirectKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + if seen.Has(cids[i]) { + // Pin was already pinned recursively + continue + } + dsPinner.PinWithMode(cids[i], ipfspinner.Direct) + } + convCount += len(cids) + + err = dsPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + // Delete root mdag key from datastore to remove old pin storage. + ipldPinDatastoreKey := ds.NewKey(ipldPinPath) + if err = dstore.Delete(ipldPinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot delete old pin state: %v", err) + } + if err = dstore.Sync(ipldPinDatastoreKey); err != nil { + return nil, 0, fmt.Errorf("cannot sync old pin state: %v", err) + } + + return dsPinner, convCount, nil +} + +// ConvertPinsFromDSToIPLD converts the pins stored in the datastore by +// dspinner, into pins stored in the given internal DAGService by ipldpinner. +// Returns an ipldpinner loaded with the converted pins, and a count of the +// recursive and direct pins converted. +// +// After the pins are stored in the DAGService, the pins and their indexes are +// removed from the dspinner. +func ConvertPinsFromDSToIPLD(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { + dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + if err != nil { + return nil, 0, err + } + + ipldPinner := ipldpinner.New(dstore, dserv, internal) + if err != nil { + return nil, 0, err + } + + cids, err := dsPinner.RecursiveKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + ipldPinner.PinWithMode(cids[i], ipfspinner.Recursive) + dsPinner.RemovePinWithMode(cids[i], ipfspinner.Recursive) + } + convCount := len(cids) + + cids, err = dsPinner.DirectKeys(ctx) + if err != nil { + return nil, 0, err + } + for i := range cids { + ipldPinner.PinWithMode(cids[i], ipfspinner.Direct) + dsPinner.RemovePinWithMode(cids[i], ipfspinner.Direct) + } + convCount += len(cids) + + // Save the ipldpinner pins + err = ipldPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + err = dsPinner.Flush(ctx) + if err != nil { + return nil, 0, err + } + + return ipldPinner, convCount, nil +} diff --git a/pinconv/pinconv_test.go b/pinconv/pinconv_test.go new file mode 100644 index 0000000..345c6a1 --- /dev/null +++ b/pinconv/pinconv_test.go @@ -0,0 +1,153 @@ +package pinconv + +import ( + "context" + "errors" + "io" + "testing" + + bs "github.com/ipfs/go-blockservice" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + lds "github.com/ipfs/go-ds-leveldb" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + ipfspin "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dspinner" + util "github.com/ipfs/go-ipfs-util" + ipld "github.com/ipfs/go-ipld-format" + mdag "github.com/ipfs/go-merkledag" +) + +var rand = util.NewTimeSeededRand() + +type batchWrap struct { + ds.Datastore +} + +func randNode() (*mdag.ProtoNode, cid.Cid) { + nd := new(mdag.ProtoNode) + nd.SetData(make([]byte, 32)) + _, err := io.ReadFull(rand, nd.Data()) + if err != nil { + panic(err) + } + k := nd.Cid() + return nd, k +} + +func (d *batchWrap) Batch() (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} + +func makeStore() (ds.Datastore, ipld.DAGService) { + ldstore, err := lds.NewDatastore("", nil) + if err != nil { + panic(err) + } + var dstore ds.Batching + dstore = &batchWrap{ldstore} + + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + dserv := mdag.NewDAGService(bserv) + return dstore, dserv +} + +func TestConversions(t *testing.T) { + ctx := context.Background() + dstore, dserv := makeStore() + + dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + a, ak := randNode() + err = dsPinner.Pin(ctx, a, false) + if err != nil { + t.Fatal(err) + } + + // create new node c, to be indirectly pinned through b + c, ck := randNode() + dserv.Add(ctx, c) + + // Create new node b, to be parent to a and c + b, _ := randNode() + b.AddNodeLink("child", a) + b.AddNodeLink("otherchild", c) + bk := b.Cid() // CID changed after adding links + + // recursively pin B{A,C} + err = dsPinner.Pin(ctx, b, true) + if err != nil { + t.Fatal(err) + } + + err = dsPinner.Flush(ctx) + if err != nil { + t.Fatal(err) + } + + verifyPins := func(pinner ipfspin.Pinner) error { + pinned, err := pinner.CheckIfPinned(ctx, ak, bk, ck) + if err != nil { + return err + } + if len(pinned) != 3 { + return errors.New("incorrect number of results") + } + for _, pn := range pinned { + switch pn.Key { + case ak: + if pn.Mode != ipfspin.Direct { + return errors.New("A pinned with wrong mode") + } + case bk: + if pn.Mode != ipfspin.Recursive { + return errors.New("B pinned with wrong mode") + } + case ck: + if pn.Mode != ipfspin.Indirect { + return errors.New("C should be pinned indirectly") + } + if pn.Via != bk { + return errors.New("C should be pinned via B") + } + } + } + return nil + } + + err = verifyPins(dsPinner) + if err != nil { + t.Fatal(err) + } + + ipldPinner, toIPLDCount, err := ConvertPinsFromDSToIPLD(ctx, dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if toIPLDCount != 2 { + t.Fatal("expected 2 ds-to-ipld pins, got", toIPLDCount) + } + + err = verifyPins(ipldPinner) + if err != nil { + t.Fatal(err) + } + + toDSPinner, toDSCount, err := ConvertPinsFromIPLDToDS(ctx, dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } + if toDSCount != toIPLDCount { + t.Fatal("ds-to-ipld pins", toIPLDCount, "not equal to ipld-to-ds-pins", toDSCount) + } + + err = verifyPins(toDSPinner) + if err != nil { + t.Fatal(err) + } +} From d787682136a5e3cd08cb9eca357754cd741c6bbd Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 03:12:25 -0800 Subject: [PATCH 17/29] Additional changes from review - Indexer functions take context - SyncIndex is not part of Indexer interface - Test corrupt index by adding index with no pin - and more... --- dsindex/indexer.go | 77 ++++++++++++----------- dsindex/indexer_test.go | 75 +++++++++++++---------- dspinner/pin.go | 131 ++++++++++++++++++++-------------------- dspinner/pin_test.go | 98 +++++++++++++++++++++--------- 4 files changed, 218 insertions(+), 163 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index ead822b..ec12321 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -2,6 +2,7 @@ package dsindex import ( + "context" "path" ds "github.com/ipfs/go-datastore" @@ -12,38 +13,33 @@ import ( // to one more primary keys. type Indexer interface { // Add adds the specified key to the an index - Add(index, id string) error + Add(ctx context.Context, index, id string) error // Delete deletes the specified key from the index. If the key is not in // the datastore, this method returns no error. - Delete(index, id string) error + Delete(ctx context.Context, index, id string) error // DeleteIndex deletes all keys in the given index. If an index is not in // the datastore, this method returns no error. - DeleteIndex(index string) (count int, err error) + DeleteIndex(ctx context.Context, index string) (count int, err error) // DeleteAll deletes all indexes managed by this Indexer - DeleteAll() (count int, err error) + DeleteAll(ctx context.Context) (count int, err error) // ForEach calls the function for each key in the specified index, until // there are no more keys, or until the function returns false. If index // is empty string, then all indexs are iterated. - ForEach(index string, fn func(index, id string) bool) error + ForEach(ctx context.Context, index string, fn func(index, id string) bool) error // HasKey determines if the index contains the specified key - HasKey(index, id string) (bool, error) + HasKey(ctx context.Context, index, id string) (bool, error) // HasAny determines if any key is in the specified index. If index is // empty string, then all indexes are searched. - HasAny(index string) (bool, error) + HasAny(ctx context.Context, index string) (bool, error) // Search returns all keys for the given index - Search(index string) (ids []string, err error) - - // Synchronize the indexes in this Indexer to match those of the given - // Indexer. The indexPath prefix is not synchronized, only the index/key - // portion of the indexes. - SyncTo(reference Indexer) (changed bool, err error) + Search(ctx context.Context, index string) (ids []string, err error) } // indexer is a simple implementation of Indexer. This implementation relies @@ -67,7 +63,7 @@ func New(dstore ds.Datastore, indexPath string) Indexer { } } -func (x *indexer) Add(index, id string) error { +func (x *indexer) Add(ctx context.Context, index, id string) error { if index == "" { return ErrEmptyIndex } @@ -78,7 +74,7 @@ func (x *indexer) Add(index, id string) error { return x.dstore.Put(key, []byte{}) } -func (x *indexer) Delete(index, id string) error { +func (x *indexer) Delete(ctx context.Context, index, id string) error { if index == "" { return ErrEmptyIndex } @@ -88,18 +84,18 @@ func (x *indexer) Delete(index, id string) error { return x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, index, id))) } -func (x *indexer) DeleteIndex(index string) (int, error) { +func (x *indexer) DeleteIndex(ctx context.Context, index string) (int, error) { if index == "" { return 0, ErrEmptyIndex } - return x.deletePrefix(path.Join(x.indexPath, index)) + return x.deletePrefix(ctx, path.Join(x.indexPath, index)) } -func (x *indexer) DeleteAll() (int, error) { - return x.deletePrefix(x.indexPath) +func (x *indexer) DeleteAll(ctx context.Context) (int, error) { + return x.deletePrefix(ctx, x.indexPath) } -func (x *indexer) ForEach(index string, fn func(idx, id string) bool) error { +func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, id string) bool) error { q := query.Query{ Prefix: path.Join(x.indexPath, index), KeysOnly: true, @@ -129,7 +125,7 @@ func (x *indexer) ForEach(index string, fn func(idx, id string) bool) error { return err } -func (x *indexer) HasKey(index, id string) (bool, error) { +func (x *indexer) HasKey(ctx context.Context, index, id string) (bool, error) { if index == "" { return false, ErrEmptyIndex } @@ -139,20 +135,20 @@ func (x *indexer) HasKey(index, id string) (bool, error) { return x.dstore.Has(ds.NewKey(path.Join(x.indexPath, index, id))) } -func (x *indexer) HasAny(index string) (bool, error) { +func (x *indexer) HasAny(ctx context.Context, index string) (bool, error) { var any bool - err := x.ForEach(index, func(idx, id string) bool { + err := x.ForEach(ctx, index, func(idx, id string) bool { any = true return false }) return any, err } -func (x *indexer) Search(index string) ([]string, error) { +func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { if index == "" { return nil, ErrEmptyIndex } - ents, err := x.queryPrefix(path.Join(x.indexPath, index)) + ents, err := x.queryPrefix(ctx, path.Join(x.indexPath, index)) if err != nil { return nil, err } @@ -167,10 +163,13 @@ func (x *indexer) Search(index string) ([]string, error) { return ids, nil } -func (x *indexer) SyncTo(ref Indexer) (bool, error) { +// SyncIndex synchronizes the indexes in the target Indexer to match those of +// the ref Indexer. The indexPath prefix is not synchronized, only the +// index/key portion of the indexes. +func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Build reference index map refs := map[string]string{} - err := ref.ForEach("", func(idx, id string) bool { + err := ref.ForEach(ctx, "", func(idx, id string) bool { refs[id] = idx return true }) @@ -182,14 +181,14 @@ func (x *indexer) SyncTo(ref Indexer) (bool, error) { } // Compare current indexes - var delKeys []string - err = x.ForEach("", func(idx, id string) bool { + dels := map[string]string{} + err = target.ForEach(ctx, "", func(idx, id string) bool { refIdx, ok := refs[id] if ok && refIdx == idx { // same in both; delete from refs, do not add to delKeys delete(refs, id) } else { - delKeys = append(delKeys, path.Join(x.indexPath, idx, id)) + dels[id] = idx } return true }) @@ -197,27 +196,27 @@ func (x *indexer) SyncTo(ref Indexer) (bool, error) { return false, err } - // Items in delKeys are indexes that no longer exist - for i := range delKeys { - err = x.dstore.Delete(ds.NewKey(delKeys[i])) + // Items in dels are indexes that no longer exist + for key, idx := range dels { + err = target.Delete(ctx, idx, key) if err != nil { return false, err } } // What remains in refs are indexes that need to be added - for k, v := range refs { - err = x.dstore.Put(ds.NewKey(path.Join(x.indexPath, v, k)), nil) + for key, idx := range refs { + err = target.Add(ctx, idx, key) if err != nil { return false, err } } - return len(refs) != 0 || len(delKeys) != 0, nil + return len(refs) != 0 || len(dels) != 0, nil } -func (x *indexer) deletePrefix(prefix string) (int, error) { - ents, err := x.queryPrefix(prefix) +func (x *indexer) deletePrefix(ctx context.Context, prefix string) (int, error) { + ents, err := x.queryPrefix(ctx, prefix) if err != nil { return 0, err } @@ -232,7 +231,7 @@ func (x *indexer) deletePrefix(prefix string) (int, error) { return len(ents), nil } -func (x *indexer) queryPrefix(prefix string) ([]query.Entry, error) { +func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry, error) { q := query.Query{ Prefix: prefix, KeysOnly: true, diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go index 72783d9..6c2f781 100644 --- a/dsindex/indexer_test.go +++ b/dsindex/indexer_test.go @@ -1,6 +1,7 @@ package dsindex import ( + "context" "testing" ds "github.com/ipfs/go-datastore" @@ -10,40 +11,43 @@ func createIndexer() Indexer { dstore := ds.NewMapDatastore() nameIndex := New(dstore, "/data/nameindex") - nameIndex.Add("alice", "a1") - nameIndex.Add("bob", "b1") - nameIndex.Add("bob", "b2") - nameIndex.Add("cathy", "c1") + ctx := context.Background() + nameIndex.Add(ctx, "alice", "a1") + nameIndex.Add(ctx, "bob", "b1") + nameIndex.Add(ctx, "bob", "b2") + nameIndex.Add(ctx, "cathy", "c1") return nameIndex } func TestAdd(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() - err := nameIndex.Add("someone", "s1") + err := nameIndex.Add(ctx, "someone", "s1") if err != nil { t.Fatal(err) } - err = nameIndex.Add("someone", "s1") + err = nameIndex.Add(ctx, "someone", "s1") if err != nil { t.Fatal(err) } - err = nameIndex.Add("", "noindex") + err = nameIndex.Add(ctx, "", "noindex") if err != ErrEmptyIndex { t.Fatal("unexpected error:", err) } - err = nameIndex.Add("nokey", "") + err = nameIndex.Add(ctx, "nokey", "") if err != ErrEmptyKey { t.Fatal("unexpected error:", err) } } func TestHasKey(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() - ok, err := nameIndex.HasKey("bob", "b1") + ok, err := nameIndex.HasKey(ctx, "bob", "b1") if err != nil { t.Fatal(err) } @@ -51,7 +55,7 @@ func TestHasKey(t *testing.T) { t.Fatal("missing index") } - ok, err = nameIndex.HasKey("bob", "b3") + ok, err = nameIndex.HasKey(ctx, "bob", "b3") if err != nil { t.Fatal(err) } @@ -59,21 +63,22 @@ func TestHasKey(t *testing.T) { t.Fatal("should not have index") } - _, err = nameIndex.HasKey("", "b1") + _, err = nameIndex.HasKey(ctx, "", "b1") if err != ErrEmptyIndex { t.Fatal("unexpected error:", err) } - _, err = nameIndex.HasKey("bob", "") + _, err = nameIndex.HasKey(ctx, "bob", "") if err != ErrEmptyKey { t.Fatal("unexpected error:", err) } } func TestHasAny(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() - ok, err := nameIndex.HasAny("nothere") + ok, err := nameIndex.HasAny(ctx, "nothere") if err != nil { t.Fatal(err) } @@ -82,7 +87,7 @@ func TestHasAny(t *testing.T) { } for _, idx := range []string{"alice", "bob", ""} { - ok, err = nameIndex.HasAny(idx) + ok, err = nameIndex.HasAny(ctx, idx) if err != nil { t.Fatal(err) } @@ -91,7 +96,7 @@ func TestHasAny(t *testing.T) { } } - count, err := nameIndex.DeleteAll() + count, err := nameIndex.DeleteAll(ctx) if err != nil { t.Fatal(err) } @@ -99,7 +104,7 @@ func TestHasAny(t *testing.T) { t.Fatal("expected 4 deletions") } - ok, err = nameIndex.HasAny("") + ok, err = nameIndex.HasAny(ctx, "") if err != nil { t.Fatal(err) } @@ -109,10 +114,11 @@ func TestHasAny(t *testing.T) { } func TestForEach(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() found := make(map[string]struct{}) - err := nameIndex.ForEach("bob", func(idx, id string) bool { + err := nameIndex.ForEach(ctx, "bob", func(idx, id string) bool { found[id] = struct{}{} return true }) @@ -128,7 +134,7 @@ func TestForEach(t *testing.T) { } keys := map[string]string{} - err = nameIndex.ForEach("", func(idx, id string) bool { + err = nameIndex.ForEach(ctx, "", func(idx, id string) bool { keys[id] = idx return true }) @@ -154,9 +160,10 @@ func TestForEach(t *testing.T) { } func TestSearch(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() - ids, err := nameIndex.Search("bob") + ids, err := nameIndex.Search(ctx, "bob") if err != nil { t.Fatal(err) } @@ -172,7 +179,7 @@ func TestSearch(t *testing.T) { t.Fatal("duplicate id") } - ids, err = nameIndex.Search("cathy") + ids, err = nameIndex.Search(ctx, "cathy") if err != nil { t.Fatal(err) } @@ -180,7 +187,7 @@ func TestSearch(t *testing.T) { t.Fatal("wrong ids") } - ids, err = nameIndex.Search("amit") + ids, err = nameIndex.Search(ctx, "amit") if err != nil { t.Fatal(err) } @@ -190,19 +197,20 @@ func TestSearch(t *testing.T) { } func TestDelete(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() - err := nameIndex.Delete("bob", "b3") + err := nameIndex.Delete(ctx, "bob", "b3") if err != nil { t.Fatal(err) } - err = nameIndex.Delete("alice", "a1") + err = nameIndex.Delete(ctx, "alice", "a1") if err != nil { t.Fatal(err) } - ok, err := nameIndex.HasKey("alice", "a1") + ok, err := nameIndex.HasKey(ctx, "alice", "a1") if err != nil { t.Fatal(err) } @@ -210,29 +218,30 @@ func TestDelete(t *testing.T) { t.Fatal("index key should have been deleted") } - count, err := nameIndex.DeleteIndex("bob") + count, err := nameIndex.DeleteIndex(ctx, "bob") if err != nil { t.Fatal(err) } if count != 2 { t.Fatal("wrong deleted count") } - ok, _ = nameIndex.HasKey("bob", "b1") + ok, _ = nameIndex.HasKey(ctx, "bob", "b1") if ok { t.Fatal("index not deleted") } } -func TestSyncTo(t *testing.T) { +func TestSyncIndex(t *testing.T) { + ctx := context.Background() nameIndex := createIndexer() dstore := ds.NewMapDatastore() refIndex := New(dstore, "/ref") - refIndex.Add("alice", "a1") - refIndex.Add("cathy", "zz") - refIndex.Add("dennis", "d1") + refIndex.Add(ctx, "alice", "a1") + refIndex.Add(ctx, "cathy", "zz") + refIndex.Add(ctx, "dennis", "d1") - changed, err := nameIndex.SyncTo(refIndex) + changed, err := SyncIndex(ctx, refIndex, nameIndex) if err != nil { t.Fatal(err) } @@ -242,7 +251,7 @@ func TestSyncTo(t *testing.T) { // Create map of id->index in sync target syncs := map[string]string{} - err = nameIndex.ForEach("", func(idx, id string) bool { + err = nameIndex.ForEach(ctx, "", func(idx, id string) bool { syncs[id] = idx return true }) @@ -252,7 +261,7 @@ func TestSyncTo(t *testing.T) { // Iterate items in sync source and make sure they appear in target var itemCount int - err = refIndex.ForEach("", func(idx, id string) bool { + err = refIndex.ForEach(ctx, "", func(idx, id string) bool { itemCount++ syncIdx, ok := syncs[id] if !ok || idx != syncIdx { diff --git a/dspinner/pin.go b/dspinner/pin.go index 986768b..5f7ba06 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -133,7 +133,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { defer p.lock.Unlock() if recurse { - found, err := p.cidRIndex.HasAny(cidStr) + found, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return err } @@ -154,7 +154,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { // Only look again if something has changed. if p.dirty != dirtyBefore { - found, err = p.cidRIndex.HasAny(cidStr) + found, err = p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return err } @@ -164,20 +164,20 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { } // TODO: remove this to support multiple pins per CID - found, err = p.cidDIndex.HasAny(cidStr) + found, err = p.cidDIndex.HasAny(ctx, cidStr) if err != nil { return err } if found { - p.removePinsForCid(c, ipfspinner.Direct) + p.removePinsForCid(ctx, c, ipfspinner.Direct) } - err = p.addPin(c, ipfspinner.Recursive, "") + err = p.addPin(ctx, c, ipfspinner.Recursive, "") if err != nil { return err } } else { - found, err := p.cidRIndex.HasAny(cidStr) + found, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return err } @@ -185,7 +185,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return fmt.Errorf("%s already pinned recursively", cidStr) } - err = p.addPin(c, ipfspinner.Direct, "") + err = p.addPin(ctx, c, ipfspinner.Direct, "") if err != nil { return err } @@ -193,7 +193,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return nil } -func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { +func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) error { // Create new pin and store in datastore pp := newPin(c, mode, name) @@ -203,14 +203,14 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { return fmt.Errorf("could not encode pin: %v", err) } - p.setDirty(true) + p.setDirty(ctx, true) // Store CID index switch mode { case ipfspinner.Recursive: - err = p.cidRIndex.Add(c.String(), pp.id) + err = p.cidRIndex.Add(ctx, c.String(), pp.id) case ipfspinner.Direct: - err = p.cidDIndex.Add(c.String(), pp.id) + err = p.cidDIndex.Add(ctx, c.String(), pp.id) default: panic("pin mode must be recursive or direct") } @@ -220,7 +220,7 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { if name != "" { // Store name index - err = p.nameIndex.Add(name, pp.id) + err = p.nameIndex.Add(ctx, name, pp.id) if err != nil { return fmt.Errorf("could not add pin name index: %v", err) } @@ -230,12 +230,12 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { if mode == ipfspinner.Recursive { - p.cidRIndex.Delete(c.String(), pp.id) + p.cidRIndex.Delete(ctx, c.String(), pp.id) } else { - p.cidDIndex.Delete(c.String(), pp.id) + p.cidDIndex.Delete(ctx, c.String(), pp.id) } if name != "" { - p.nameIndex.Delete(name, pp.id) + p.nameIndex.Delete(ctx, name, pp.id) } return err } @@ -243,8 +243,8 @@ func (p *pinner) addPin(c cid.Cid, mode ipfspinner.Mode, name string) error { return nil } -func (p *pinner) removePin(pp *pin) error { - p.setDirty(true) +func (p *pinner) removePin(ctx context.Context, pp *pin) error { + p.setDirty(ctx, true) // Remove pin from datastore. Pin must be removed before index for // recovery to work. @@ -254,9 +254,9 @@ func (p *pinner) removePin(pp *pin) error { } // Remove cid index from datastore if pp.mode == ipfspinner.Recursive { - err = p.cidRIndex.Delete(pp.cid.String(), pp.id) + err = p.cidRIndex.Delete(ctx, pp.cid.String(), pp.id) } else { - err = p.cidDIndex.Delete(pp.cid.String(), pp.id) + err = p.cidDIndex.Delete(ctx, pp.cid.String(), pp.id) } if err != nil { return err @@ -264,7 +264,7 @@ func (p *pinner) removePin(pp *pin) error { if pp.name != "" { // Remove name index from datastore - err = p.nameIndex.Delete(pp.name, pp.id) + err = p.nameIndex.Delete(ctx, pp.name, pp.id) if err != nil { return err } @@ -287,7 +287,7 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } matches := p.Ls(matchSpec) */ - has, err := p.cidRIndex.HasAny(cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return err } @@ -297,7 +297,7 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { return fmt.Errorf("%s is pinned recursively", c) } } else { - has, err = p.cidDIndex.HasAny(cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidStr) if err != nil { return err } @@ -306,7 +306,7 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } } - _, err = p.removePinsForCid(c, ipfspinner.Any) + _, err = p.removePinsForCid(ctx, c, ipfspinner.Any) if err != nil { return err } @@ -334,7 +334,7 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne cidStr := c.String() switch mode { case ipfspinner.Recursive: - has, err := p.cidRIndex.HasAny(cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return "", false, err } @@ -343,7 +343,7 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne } return "", false, nil case ipfspinner.Direct: - has, err := p.cidDIndex.HasAny(cidStr) + has, err := p.cidDIndex.HasAny(ctx, cidStr) if err != nil { return "", false, err } @@ -355,14 +355,14 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne return "", false, nil case ipfspinner.Indirect: case ipfspinner.Any: - has, err := p.cidRIndex.HasAny(cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return "", false, err } if has { return linkRecursive, true, nil } - has, err = p.cidDIndex.HasAny(cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidStr) if err != nil { return "", false, err } @@ -384,7 +384,7 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne var has bool var rc cid.Cid var e error - err := p.cidRIndex.ForEach("", func(idx, id string) bool { + err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { rc, e = cid.Decode(idx) if e != nil { return false @@ -426,14 +426,14 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn // First check for non-Indirect pins directly for _, c := range cids { cidStr := c.String() - has, err := p.cidRIndex.HasAny(cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidStr) if err != nil { return nil, err } if has { pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) } else { - has, err = p.cidDIndex.HasAny(cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidStr) if err != nil { return nil, err } @@ -474,7 +474,7 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn } var e error - err := p.cidRIndex.ForEach("", func(idx, id string) bool { + err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { var rk cid.Cid rk, e = cid.Decode(idx) if e != nil { @@ -508,6 +508,7 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn // Use with care! If used improperly, garbage collection may not // be successful. func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { + ctx := context.TODO() // Check cache to see if CID is pinned switch mode { case ipfspinner.Direct, ipfspinner.Recursive: @@ -519,28 +520,28 @@ func (p *pinner) RemovePinWithMode(c cid.Cid, mode ipfspinner.Mode) { p.lock.Lock() defer p.lock.Unlock() - p.removePinsForCid(c, mode) + p.removePinsForCid(ctx, c, mode) } // removePinsForCid removes all pins for a cid that has the specified mode. // Returns true if any pins, and all corresponding CID index entries, were // removed. Otherwise, returns false. -func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) { +func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (bool, error) { // Search for pins by CID var ids []string var err error cidStr := c.String() switch mode { case ipfspinner.Recursive: - ids, err = p.cidRIndex.Search(cidStr) + ids, err = p.cidRIndex.Search(ctx, cidStr) case ipfspinner.Direct: - ids, err = p.cidDIndex.Search(cidStr) + ids, err = p.cidDIndex.Search(ctx, cidStr) case ipfspinner.Any: - ids, err = p.cidRIndex.Search(cidStr) + ids, err = p.cidRIndex.Search(ctx, cidStr) if err != nil { return false, err } - dIds, err := p.cidDIndex.Search(cidStr) + dIds, err := p.cidDIndex.Search(ctx, cidStr) if err != nil { return false, err } @@ -557,19 +558,19 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) // Remove the pin with the requested mode for _, pid := range ids { var pp *pin - pp, err = p.loadPin(pid) + pp, err = p.loadPin(ctx, pid) if err != nil { if err == ds.ErrNotFound { - p.setDirty(true) + p.setDirty(ctx, true) // Fix index; remove index for pin that does not exist switch mode { case ipfspinner.Recursive: - p.cidRIndex.DeleteIndex(cidStr) + p.cidRIndex.DeleteIndex(ctx, cidStr) case ipfspinner.Direct: - p.cidDIndex.DeleteIndex(cidStr) + p.cidDIndex.DeleteIndex(ctx, cidStr) case ipfspinner.Any: - p.cidRIndex.DeleteIndex(cidStr) - p.cidDIndex.DeleteIndex(cidStr) + p.cidRIndex.DeleteIndex(ctx, cidStr) + p.cidDIndex.DeleteIndex(ctx, cidStr) } log.Error("found CID index with missing pin") continue @@ -577,7 +578,7 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) return false, err } if mode == ipfspinner.Any || pp.mode == mode { - err = p.removePin(pp) + err = p.removePin(ctx, pp) if err != nil { return false, err } @@ -588,7 +589,7 @@ func (p *pinner) removePinsForCid(c cid.Cid, mode ipfspinner.Mode) (bool, error) } // loadPin loads a single pin from the datastore. -func (p *pinner) loadPin(pid string) (*pin, error) { +func (p *pinner) loadPin(ctx context.Context, pid string) (*pin, error) { pinData, err := p.dstore.Get(ds.NewKey(path.Join(pinKeyPath, pid))) if err != nil { return nil, err @@ -597,7 +598,7 @@ func (p *pinner) loadPin(pid string) (*pin, error) { } // loadAllPins loads all pins from the datastore. -func (p *pinner) loadAllPins() ([]*pin, error) { +func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { q := query.Query{ Prefix: pinKeyPath, } @@ -639,7 +640,7 @@ func LoadPinner(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) if data[0] == 1 { p.dirty = 1 - pins, err := p.loadAllPins() + pins, err := p.loadAllPins(ctx) if err != nil { return nil, fmt.Errorf("cannot load pins: %v", err) } @@ -665,12 +666,12 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { var hasNames bool for _, pp := range pins { if pp.mode == ipfspinner.Recursive { - tmpCidRIndex.Add(pp.cid.String(), pp.id) + tmpCidRIndex.Add(ctx, pp.cid.String(), pp.id) } else if pp.mode == ipfspinner.Direct { - tmpCidDIndex.Add(pp.cid.String(), pp.id) + tmpCidDIndex.Add(ctx, pp.cid.String(), pp.id) } if pp.name != "" { - tmpNameIndex.Add(pp.name, pp.id) + tmpNameIndex.Add(ctx, pp.name, pp.id) hasNames = true } } @@ -678,7 +679,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { // Sync the CID index to what was build from pins. This fixes any invalid // indexes, which could happen if ipfs was terminated between writing pin // and writing secondary index. - changed, err := p.cidRIndex.SyncTo(tmpCidRIndex) + changed, err := dsindex.SyncIndex(ctx, tmpCidRIndex, p.cidRIndex) if err != nil { return fmt.Errorf("cannot sync indexes: %v", err) } @@ -686,7 +687,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { log.Info("invalid recursive indexes detected - rebuilt") } - changed, err = p.cidDIndex.SyncTo(tmpCidDIndex) + changed, err = dsindex.SyncIndex(ctx, tmpCidDIndex, p.cidDIndex) if err != nil { return fmt.Errorf("cannot sync indexes: %v", err) } @@ -695,7 +696,7 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { } if hasNames { - changed, err = p.nameIndex.SyncTo(tmpNameIndex) + changed, err = dsindex.SyncIndex(ctx, tmpNameIndex, p.nameIndex) if err != nil { return fmt.Errorf("cannot sync name indexes: %v", err) } @@ -714,7 +715,7 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { cidSet := cid.NewSet() var e error - err := p.cidDIndex.ForEach("", func(idx, id string) bool { + err := p.cidDIndex.ForEach(ctx, "", func(idx, id string) bool { var c cid.Cid c, e = cid.Decode(idx) if e != nil { @@ -740,7 +741,7 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { cidSet := cid.NewSet() var e error - err := p.cidRIndex.ForEach("", func(idx, id string) bool { + err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { var c cid.Cid c, e = cid.Decode(idx) if e != nil { @@ -773,7 +774,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error p.lock.Lock() defer p.lock.Unlock() - found, err := p.cidRIndex.HasAny(from.String()) + found, err := p.cidRIndex.HasAny(ctx, from.String()) if err != nil { return err } @@ -787,7 +788,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error } // Check if the `to` cid is already recursively pinned - found, err = p.cidRIndex.HasAny(to.String()) + found, err = p.cidRIndex.HasAny(ctx, to.String()) if err != nil { return err } @@ -804,7 +805,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return err } - err = p.addPin(to, ipfspinner.Recursive, "") + err = p.addPin(ctx, to, ipfspinner.Recursive, "") if err != nil { return err } @@ -813,7 +814,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return nil } - _, err = p.removePinsForCid(from, ipfspinner.Recursive) + _, err = p.removePinsForCid(ctx, from, ipfspinner.Recursive) if err != nil { return err } @@ -837,7 +838,7 @@ func (p *pinner) Flush(ctx context.Context) error { return fmt.Errorf("cannot sync pin state: %v", err) } - p.setDirty(false) + p.setDirty(ctx, false) return nil } @@ -845,24 +846,26 @@ func (p *pinner) Flush(ctx context.Context) error { // PinWithMode allows the user to have fine grained control over pin // counts func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { + ctx := context.TODO() + p.lock.Lock() defer p.lock.Unlock() // TODO: remove his to support multiple pins per CID switch mode { case ipfspinner.Recursive: - if has, _ := p.cidRIndex.HasAny(c.String()); has { + if has, _ := p.cidRIndex.HasAny(ctx, c.String()); has { return // already a recursive pin for this CID } case ipfspinner.Direct: - if has, _ := p.cidDIndex.HasAny(c.String()); has { + if has, _ := p.cidDIndex.HasAny(ctx, c.String()); has { return // already a direct pin for this CID } default: panic("unrecognized pin mode") } - err := p.addPin(c, mode, "") + err := p.addPin(ctx, c, mode, "") if err != nil { return } @@ -977,7 +980,7 @@ func decodePin(pid string, data []byte) (*pin, error) { // setDirty saves a boolean dirty flag in the datastore whenever there is a // transition between a dirty (counter > 0) and non-dirty (counter == 0) state. -func (p *pinner) setDirty(dirty bool) { +func (p *pinner) setDirty(ctx context.Context, dirty bool) { isClean := p.dirty == p.clean if dirty { p.dirty++ diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index ee0ac44..4bc15e6 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -105,10 +105,13 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } a, ak := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -279,14 +282,14 @@ func TestPinnerBasic(t *testing.T) { // Remove the pin but not the index to simulate corruption dsp := p.(*pinner) - ids, err := dsp.cidDIndex.Search(ak.String()) + ids, err := dsp.cidDIndex.Search(ctx, ak.String()) if err != nil { t.Fatal(err) } if len(ids) == 0 { t.Fatal("did not find pin for cid", ak.String()) } - pp, err := dsp.loadPin(ids[0]) + pp, err := dsp.loadPin(ctx, ids[0]) if err != nil { t.Fatal(err) } @@ -321,14 +324,17 @@ func TestRemovePinWithMode(t *testing.T) { dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } a, ak := randNode() dserv.Add(ctx, a) p.Pin(ctx, a, false) - ok, err := p.(*pinner).removePinsForCid(ak, ipfspin.Recursive) + ok, err := p.(*pinner).removePinsForCid(ctx, ak, ipfspin.Recursive) if err != nil { t.Fatal(err) } @@ -461,11 +467,14 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() b, _ := randNode() - err := a.AddNodeLink("child", b) + err = a.AddNodeLink("child", b) if err != nil { t.Fatal(err) } @@ -506,13 +515,15 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } n1, c1 := randNode() n2, c2 := randNode() _, c3 := randNode() - err := dserv.Add(ctx, n1) - if err != nil { + if err = dserv.Add(ctx, n1); err != nil { t.Fatal(err) } if err = dserv.Add(ctx, n2); err != nil { @@ -571,25 +582,31 @@ func TestLoadDirty(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv) + p, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } a, ak := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } - // Pin A{} recursive + _, bk := randNode() + err = p.Pin(ctx, a, true) if err != nil { t.Fatal(err) } - cidStr := ak.String() + cidAStr := ak.String() + cidBStr := bk.String() // Corrupt index cidRIndex := p.(*pinner).cidRIndex - cidRIndex.DeleteIndex(cidStr) + cidRIndex.DeleteIndex(ctx, cidAStr) + cidRIndex.Add(ctx, cidBStr, "not-a-pin-id") // Verify dirty data, err := dstore.Get(dirtyKey) @@ -600,7 +617,7 @@ func TestLoadDirty(t *testing.T) { t.Fatal("dirty flag not set") } - has, err := cidRIndex.HasAny(cidStr) + has, err := cidRIndex.HasAny(ctx, cidAStr) if err != nil { t.Fatal(err) } @@ -625,13 +642,21 @@ func TestLoadDirty(t *testing.T) { // Verify index rebuilt cidRIndex = p.(*pinner).cidRIndex - has, err = cidRIndex.HasAny(cidStr) + has, err = cidRIndex.HasAny(ctx, cidAStr) if err != nil { t.Fatal(err) } if !has { t.Fatal("index should have been rebuilt") } + + has, err = cidRIndex.HasAny(ctx, cidBStr) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatal("index should have been removed by rebuild") + } } func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { @@ -779,19 +804,22 @@ func makeStore() (ds.Datastore, ipld.DAGService) { // compares the load time when rebuilding indexes to loading without rebuilding // indexes. func BenchmarkLoadRebuild(b *testing.B) { + ctx := context.Background() + dstore, dserv := makeStore() - pinner := New(dstore, dserv) + pinner, err := LoadPinner(ctx, dstore, dserv) + if err != nil { + panic(err.Error()) + } nodes := makeNodes(4096, dserv) pinNodes(nodes, pinner, true) - ctx := context.Background() - b.Run("RebuildTrue", func(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{1}) - _, err := LoadPinner(ctx, dstore, dserv) + _, err = LoadPinner(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -802,7 +830,7 @@ func BenchmarkLoadRebuild(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{0}) - _, err := LoadPinner(ctx, dstore, dserv) + _, err = LoadPinner(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -815,7 +843,10 @@ func BenchmarkLoadRebuild(b *testing.B) { // creating a pin in a larger number of existing pins. func BenchmarkNthPin(b *testing.B) { dstore, dserv := makeStore() - pinner := New(dstore, dserv) + pinner, err := LoadPinner(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } pinnerIPLD := ipldpinner.New(dstore, dserv, dserv) for count := 1000; count <= 10000; count += 1000 { @@ -852,6 +883,10 @@ func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld. if err != nil { panic(err) } + err = pinner.Flush(ctx) + if err != nil { + panic(err) + } b.StartTimer() } } @@ -862,7 +897,10 @@ func BenchmarkNPins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := New(dstore, dserv) + pinner, err := LoadPinner(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } benchmarkNPins(b, count, pinner, dserv) }) @@ -905,7 +943,10 @@ func BenchmarkNUnpins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := New(dstore, dserv) + pinner, err := LoadPinner(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } benchmarkNUnpins(b, count, pinner, dserv) }) @@ -948,7 +989,10 @@ func BenchmarkPinAll(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := New(dstore, dserv) + pinner, err := LoadPinner(context.Background(), dstore, dserv) + if err != nil { + panic(err) + } benchmarkPinAll(b, count, pinner, dserv) }) From 0435ac496505ecee6f1c5524f26c38a94fb0d337 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 03:53:03 -0800 Subject: [PATCH 18/29] Removed New in favor of only having LoadPinner --- dspinner/pin.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 5f7ba06..bdaaddf 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -108,17 +108,6 @@ type syncDAGService interface { Sync() error } -// New creates a new pinner using the given datastore as a backend -func New(dstore ds.Datastore, serv ipld.DAGService) ipfspinner.Pinner { - return &pinner{ - cidDIndex: dsindex.New(dstore, pinCidDIndexPath), - cidRIndex: dsindex.New(dstore, pinCidRIndexPath), - nameIndex: dsindex.New(dstore, pinNameIndexPath), - dserv: serv, - dstore: dstore, - } -} - // Pin the given node, optionally recursive func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { err := p.dserv.Add(ctx, node) @@ -628,7 +617,13 @@ func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { // LoadPinner loads a pinner and its keysets from the given datastore func LoadPinner(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { - p := New(dstore, dserv).(*pinner) + p := &pinner{ + cidDIndex: dsindex.New(dstore, pinCidDIndexPath), + cidRIndex: dsindex.New(dstore, pinCidRIndexPath), + nameIndex: dsindex.New(dstore, pinNameIndexPath), + dserv: dserv, + dstore: dstore, + } data, err := dstore.Get(dirtyKey) if err != nil { From dde41e91cfdc5f1994665021af505815195c31a9 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 12:32:50 -0800 Subject: [PATCH 19/29] Indexer encodes index and key to allow arbitrary strings Base36 encode the index and strings to allow them to contain any characters without interferring with the datastore key path. Base36 was chosed because it is slightly more compact than Base32, more portable than Base58, Base64, etc., and because it has a very fast implementation. dspinner can now use cid.KeyString() to store the raw byte string as in index. This avoids having to encode the cid every time it is used as an index. --- dsindex/indexer.go | 91 ++++++++++++++++++++++++++++++-------------- dspinner/pin.go | 86 ++++++++++++++++++++--------------------- dspinner/pin_test.go | 16 ++++---- go.mod | 1 + go.sum | 2 + 5 files changed, 117 insertions(+), 79 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index ec12321..1d57d81 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -3,21 +3,23 @@ package dsindex import ( "context" + "fmt" "path" ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" + "github.com/multiformats/go-base36" ) // Indexer maintains a secondary index. Each value of the secondary index maps // to one more primary keys. type Indexer interface { // Add adds the specified key to the an index - Add(ctx context.Context, index, id string) error + Add(ctx context.Context, index, key string) error // Delete deletes the specified key from the index. If the key is not in // the datastore, this method returns no error. - Delete(ctx context.Context, index, id string) error + Delete(ctx context.Context, index, key string) error // DeleteIndex deletes all keys in the given index. If an index is not in // the datastore, this method returns no error. @@ -29,10 +31,10 @@ type Indexer interface { // ForEach calls the function for each key in the specified index, until // there are no more keys, or until the function returns false. If index // is empty string, then all indexs are iterated. - ForEach(ctx context.Context, index string, fn func(index, id string) bool) error + ForEach(ctx context.Context, index string, fn func(index, key string) bool) error // HasKey determines if the index contains the specified key - HasKey(ctx context.Context, index, id string) (bool, error) + HasKey(ctx context.Context, index, key string) (bool, error) // HasAny determines if any key is in the specified index. If index is // empty string, then all indexes are searched. @@ -63,39 +65,43 @@ func New(dstore ds.Datastore, indexPath string) Indexer { } } -func (x *indexer) Add(ctx context.Context, index, id string) error { +func (x *indexer) Add(ctx context.Context, index, key string) error { if index == "" { return ErrEmptyIndex } - if id == "" { + if key == "" { return ErrEmptyKey } - key := ds.NewKey(path.Join(x.indexPath, index, id)) - return x.dstore.Put(key, []byte{}) + dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) + return x.dstore.Put(dskey, []byte{}) } -func (x *indexer) Delete(ctx context.Context, index, id string) error { +func (x *indexer) Delete(ctx context.Context, index, key string) error { if index == "" { return ErrEmptyIndex } - if id == "" { + if key == "" { return ErrEmptyKey } - return x.dstore.Delete(ds.NewKey(path.Join(x.indexPath, index, id))) + dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) + return x.dstore.Delete(dskey) } func (x *indexer) DeleteIndex(ctx context.Context, index string) (int, error) { if index == "" { return 0, ErrEmptyIndex } - return x.deletePrefix(ctx, path.Join(x.indexPath, index)) + return x.deletePrefix(ctx, path.Join(x.indexPath, encode(index))) } func (x *indexer) DeleteAll(ctx context.Context) (int, error) { return x.deletePrefix(ctx, x.indexPath) } -func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, id string) bool) error { +func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, key string) bool) error { + if index != "" { + index = encode(index) + } q := query.Query{ Prefix: path.Join(x.indexPath, index), KeysOnly: true, @@ -116,7 +122,17 @@ func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, id str } ent := r.Entry - if !fn(path.Base(path.Dir(ent.Key)), path.Base(ent.Key)) { + decIdx, err := decode(path.Base(path.Dir(ent.Key))) + if err != nil { + err = fmt.Errorf("cannot decode index: %v", err) + break + } + decKey, err := decode(path.Base(ent.Key)) + if err != nil { + err = fmt.Errorf("cannot decode key: %v", err) + break + } + if !fn(decIdx, decKey) { break } } @@ -125,19 +141,20 @@ func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, id str return err } -func (x *indexer) HasKey(ctx context.Context, index, id string) (bool, error) { +func (x *indexer) HasKey(ctx context.Context, index, key string) (bool, error) { if index == "" { return false, ErrEmptyIndex } - if id == "" { + if key == "" { return false, ErrEmptyKey } - return x.dstore.Has(ds.NewKey(path.Join(x.indexPath, index, id))) + dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) + return x.dstore.Has(dskey) } func (x *indexer) HasAny(ctx context.Context, index string) (bool, error) { var any bool - err := x.ForEach(ctx, index, func(idx, id string) bool { + err := x.ForEach(ctx, index, func(idx, key string) bool { any = true return false }) @@ -148,7 +165,7 @@ func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { if index == "" { return nil, ErrEmptyIndex } - ents, err := x.queryPrefix(ctx, path.Join(x.indexPath, index)) + ents, err := x.queryPrefix(ctx, path.Join(x.indexPath, encode(index))) if err != nil { return nil, err } @@ -156,11 +173,14 @@ func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { return nil, nil } - ids := make([]string, len(ents)) + keys := make([]string, len(ents)) for i := range ents { - ids[i] = path.Base(ents[i].Key) + keys[i], err = decode(path.Base(ents[i].Key)) + if err != nil { + return nil, fmt.Errorf("cannot decode key: %v", err) + } } - return ids, nil + return keys, nil } // SyncIndex synchronizes the indexes in the target Indexer to match those of @@ -169,8 +189,8 @@ func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Build reference index map refs := map[string]string{} - err := ref.ForEach(ctx, "", func(idx, id string) bool { - refs[id] = idx + err := ref.ForEach(ctx, "", func(idx, key string) bool { + refs[key] = idx return true }) if err != nil { @@ -182,13 +202,13 @@ func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Compare current indexes dels := map[string]string{} - err = target.ForEach(ctx, "", func(idx, id string) bool { - refIdx, ok := refs[id] + err = target.ForEach(ctx, "", func(idx, key string) bool { + refIdx, ok := refs[key] if ok && refIdx == idx { // same in both; delete from refs, do not add to delKeys - delete(refs, id) + delete(refs, key) } else { - dels[id] = idx + dels[key] = idx } return true }) @@ -242,3 +262,18 @@ func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry } return results.Rest() } + +func encode(data string) string { + if data == "" { + return "" + } + return base36.EncodeToStringUc([]byte(data)) +} + +func decode(data string) (string, error) { + b, err := base36.DecodeString(data) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/dspinner/pin.go b/dspinner/pin.go index bdaaddf..865877c 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -116,13 +116,13 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { } c := node.Cid() - cidStr := c.String() + cidKey := c.KeyString() p.lock.Lock() defer p.lock.Unlock() if recurse { - found, err := p.cidRIndex.HasAny(ctx, cidStr) + found, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return err } @@ -143,7 +143,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { // Only look again if something has changed. if p.dirty != dirtyBefore { - found, err = p.cidRIndex.HasAny(ctx, cidStr) + found, err = p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return err } @@ -153,7 +153,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { } // TODO: remove this to support multiple pins per CID - found, err = p.cidDIndex.HasAny(ctx, cidStr) + found, err = p.cidDIndex.HasAny(ctx, cidKey) if err != nil { return err } @@ -166,12 +166,12 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return err } } else { - found, err := p.cidRIndex.HasAny(ctx, cidStr) + found, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return err } if found { - return fmt.Errorf("%s already pinned recursively", cidStr) + return fmt.Errorf("%s already pinned recursively", c.String()) } err = p.addPin(ctx, c, ipfspinner.Direct, "") @@ -197,9 +197,9 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na // Store CID index switch mode { case ipfspinner.Recursive: - err = p.cidRIndex.Add(ctx, c.String(), pp.id) + err = p.cidRIndex.Add(ctx, c.KeyString(), pp.id) case ipfspinner.Direct: - err = p.cidDIndex.Add(ctx, c.String(), pp.id) + err = p.cidDIndex.Add(ctx, c.KeyString(), pp.id) default: panic("pin mode must be recursive or direct") } @@ -219,9 +219,9 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { if mode == ipfspinner.Recursive { - p.cidRIndex.Delete(ctx, c.String(), pp.id) + p.cidRIndex.Delete(ctx, c.KeyString(), pp.id) } else { - p.cidDIndex.Delete(ctx, c.String(), pp.id) + p.cidDIndex.Delete(ctx, c.KeyString(), pp.id) } if name != "" { p.nameIndex.Delete(ctx, name, pp.id) @@ -243,9 +243,9 @@ func (p *pinner) removePin(ctx context.Context, pp *pin) error { } // Remove cid index from datastore if pp.mode == ipfspinner.Recursive { - err = p.cidRIndex.Delete(ctx, pp.cid.String(), pp.id) + err = p.cidRIndex.Delete(ctx, pp.cid.KeyString(), pp.id) } else { - err = p.cidDIndex.Delete(ctx, pp.cid.String(), pp.id) + err = p.cidDIndex.Delete(ctx, pp.cid.KeyString(), pp.id) } if err != nil { return err @@ -264,7 +264,7 @@ func (p *pinner) removePin(ctx context.Context, pp *pin) error { // Unpin a given key func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { - cidStr := c.String() + cidKey := c.KeyString() p.lock.Lock() defer p.lock.Unlock() @@ -276,17 +276,17 @@ func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error { } matches := p.Ls(matchSpec) */ - has, err := p.cidRIndex.HasAny(ctx, cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return err } if has { if !recursive { - return fmt.Errorf("%s is pinned recursively", c) + return fmt.Errorf("%s is pinned recursively", c.String()) } } else { - has, err = p.cidDIndex.HasAny(ctx, cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidKey) if err != nil { return err } @@ -320,10 +320,10 @@ func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne } func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) { - cidStr := c.String() + cidKey := c.KeyString() switch mode { case ipfspinner.Recursive: - has, err := p.cidRIndex.HasAny(ctx, cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return "", false, err } @@ -332,7 +332,7 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne } return "", false, nil case ipfspinner.Direct: - has, err := p.cidDIndex.HasAny(ctx, cidStr) + has, err := p.cidDIndex.HasAny(ctx, cidKey) if err != nil { return "", false, err } @@ -344,14 +344,14 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne return "", false, nil case ipfspinner.Indirect: case ipfspinner.Any: - has, err := p.cidRIndex.HasAny(ctx, cidStr) + has, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return "", false, err } if has { return linkRecursive, true, nil } - has, err = p.cidDIndex.HasAny(ctx, cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidKey) if err != nil { return "", false, err } @@ -374,7 +374,7 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne var rc cid.Cid var e error err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { - rc, e = cid.Decode(idx) + rc, e = cid.Cast([]byte(idx)) if e != nil { return false } @@ -414,15 +414,15 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn // First check for non-Indirect pins directly for _, c := range cids { - cidStr := c.String() - has, err := p.cidRIndex.HasAny(ctx, cidStr) + cidKey := c.KeyString() + has, err := p.cidRIndex.HasAny(ctx, cidKey) if err != nil { return nil, err } if has { pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive}) } else { - has, err = p.cidDIndex.HasAny(ctx, cidStr) + has, err = p.cidDIndex.HasAny(ctx, cidKey) if err != nil { return nil, err } @@ -465,7 +465,7 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn var e error err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { var rk cid.Cid - rk, e = cid.Decode(idx) + rk, e = cid.Cast([]byte(idx)) if e != nil { return false } @@ -519,18 +519,18 @@ func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinne // Search for pins by CID var ids []string var err error - cidStr := c.String() + cidKey := c.KeyString() switch mode { case ipfspinner.Recursive: - ids, err = p.cidRIndex.Search(ctx, cidStr) + ids, err = p.cidRIndex.Search(ctx, cidKey) case ipfspinner.Direct: - ids, err = p.cidDIndex.Search(ctx, cidStr) + ids, err = p.cidDIndex.Search(ctx, cidKey) case ipfspinner.Any: - ids, err = p.cidRIndex.Search(ctx, cidStr) + ids, err = p.cidRIndex.Search(ctx, cidKey) if err != nil { return false, err } - dIds, err := p.cidDIndex.Search(ctx, cidStr) + dIds, err := p.cidDIndex.Search(ctx, cidKey) if err != nil { return false, err } @@ -554,12 +554,12 @@ func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinne // Fix index; remove index for pin that does not exist switch mode { case ipfspinner.Recursive: - p.cidRIndex.DeleteIndex(ctx, cidStr) + p.cidRIndex.DeleteIndex(ctx, cidKey) case ipfspinner.Direct: - p.cidDIndex.DeleteIndex(ctx, cidStr) + p.cidDIndex.DeleteIndex(ctx, cidKey) case ipfspinner.Any: - p.cidRIndex.DeleteIndex(ctx, cidStr) - p.cidDIndex.DeleteIndex(ctx, cidStr) + p.cidRIndex.DeleteIndex(ctx, cidKey) + p.cidDIndex.DeleteIndex(ctx, cidKey) } log.Error("found CID index with missing pin") continue @@ -661,9 +661,9 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { var hasNames bool for _, pp := range pins { if pp.mode == ipfspinner.Recursive { - tmpCidRIndex.Add(ctx, pp.cid.String(), pp.id) + tmpCidRIndex.Add(ctx, pp.cid.KeyString(), pp.id) } else if pp.mode == ipfspinner.Direct { - tmpCidDIndex.Add(ctx, pp.cid.String(), pp.id) + tmpCidDIndex.Add(ctx, pp.cid.KeyString(), pp.id) } if pp.name != "" { tmpNameIndex.Add(ctx, pp.name, pp.id) @@ -712,7 +712,7 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { var e error err := p.cidDIndex.ForEach(ctx, "", func(idx, id string) bool { var c cid.Cid - c, e = cid.Decode(idx) + c, e = cid.Cast([]byte(idx)) if e != nil { return false } @@ -738,7 +738,7 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { var e error err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { var c cid.Cid - c, e = cid.Decode(idx) + c, e = cid.Cast([]byte(idx)) if e != nil { return false } @@ -769,7 +769,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error p.lock.Lock() defer p.lock.Unlock() - found, err := p.cidRIndex.HasAny(ctx, from.String()) + found, err := p.cidRIndex.HasAny(ctx, from.KeyString()) if err != nil { return err } @@ -783,7 +783,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error } // Check if the `to` cid is already recursively pinned - found, err = p.cidRIndex.HasAny(ctx, to.String()) + found, err = p.cidRIndex.HasAny(ctx, to.KeyString()) if err != nil { return err } @@ -849,11 +849,11 @@ func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { // TODO: remove his to support multiple pins per CID switch mode { case ipfspinner.Recursive: - if has, _ := p.cidRIndex.HasAny(ctx, c.String()); has { + if has, _ := p.cidRIndex.HasAny(ctx, c.KeyString()); has { return // already a recursive pin for this CID } case ipfspinner.Direct: - if has, _ := p.cidDIndex.HasAny(ctx, c.String()); has { + if has, _ := p.cidDIndex.HasAny(ctx, c.KeyString()); has { return // already a direct pin for this CID } default: diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 4bc15e6..02799c1 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -282,7 +282,7 @@ func TestPinnerBasic(t *testing.T) { // Remove the pin but not the index to simulate corruption dsp := p.(*pinner) - ids, err := dsp.cidDIndex.Search(ctx, ak.String()) + ids, err := dsp.cidDIndex.Search(ctx, ak.KeyString()) if err != nil { t.Fatal(err) } @@ -600,13 +600,13 @@ func TestLoadDirty(t *testing.T) { t.Fatal(err) } - cidAStr := ak.String() - cidBStr := bk.String() + cidAKey := ak.KeyString() + cidBKey := bk.KeyString() // Corrupt index cidRIndex := p.(*pinner).cidRIndex - cidRIndex.DeleteIndex(ctx, cidAStr) - cidRIndex.Add(ctx, cidBStr, "not-a-pin-id") + cidRIndex.DeleteIndex(ctx, cidAKey) + cidRIndex.Add(ctx, cidBKey, "not-a-pin-id") // Verify dirty data, err := dstore.Get(dirtyKey) @@ -617,7 +617,7 @@ func TestLoadDirty(t *testing.T) { t.Fatal("dirty flag not set") } - has, err := cidRIndex.HasAny(ctx, cidAStr) + has, err := cidRIndex.HasAny(ctx, cidAKey) if err != nil { t.Fatal(err) } @@ -642,7 +642,7 @@ func TestLoadDirty(t *testing.T) { // Verify index rebuilt cidRIndex = p.(*pinner).cidRIndex - has, err = cidRIndex.HasAny(ctx, cidAStr) + has, err = cidRIndex.HasAny(ctx, cidAKey) if err != nil { t.Fatal(err) } @@ -650,7 +650,7 @@ func TestLoadDirty(t *testing.T) { t.Fatal("index should have been rebuilt") } - has, err = cidRIndex.HasAny(ctx, cidBStr) + has, err = cidRIndex.HasAny(ctx, cidBKey) if err != nil { t.Fatal(err) } diff --git a/go.mod b/go.mod index 30a08c9..d7e25fd 100644 --- a/go.mod +++ b/go.mod @@ -14,5 +14,6 @@ require ( github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.1 github.com/ipfs/go-merkledag v0.3.0 + github.com/multiformats/go-base36 v0.1.0 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 ) diff --git a/go.sum b/go.sum index 86e56c0..7760462 100644 --- a/go.sum +++ b/go.sum @@ -255,6 +255,8 @@ github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= From 0a5737f38f5078e9dc3a664a8b25f3ebca7e2d7c Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 13:00:57 -0800 Subject: [PATCH 20/29] Use int64 for dirty count and remove unused const --- dspinner/pin.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 865877c..4dec4b4 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -9,7 +9,6 @@ import ( "fmt" "path" "sync" - "time" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -24,8 +23,6 @@ import ( ) const ( - loadTimeout = 5 * time.Second - basePath = "/pins" pinKeyPath = "/pins/pin" indexKeyPath = "/pins/index" @@ -76,8 +73,8 @@ type pinner struct { cidRIndex dsindex.Indexer nameIndex dsindex.Indexer - clean int - dirty int + clean int64 + dirty int64 } var _ ipfspinner.Pinner = (*pinner)(nil) From fb419e7b82a825768eceebbe420af459b670a5ed Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 16:35:58 -0800 Subject: [PATCH 21/29] use base64 encoding --- dsindex/indexer.go | 6 +++--- go.mod | 1 - go.sum | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 1d57d81..485d59b 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -3,12 +3,12 @@ package dsindex import ( "context" + "encoding/base64" "fmt" "path" ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" - "github.com/multiformats/go-base36" ) // Indexer maintains a secondary index. Each value of the secondary index maps @@ -267,11 +267,11 @@ func encode(data string) string { if data == "" { return "" } - return base36.EncodeToStringUc([]byte(data)) + return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString([]byte(data)) } func decode(data string) (string, error) { - b, err := base36.DecodeString(data) + b, err := base64.URLEncoding.WithPadding(base64.NoPadding).DecodeString(data) if err != nil { return "", err } diff --git a/go.mod b/go.mod index d7e25fd..30a08c9 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,5 @@ require ( github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log v0.0.1 github.com/ipfs/go-merkledag v0.3.0 - github.com/multiformats/go-base36 v0.1.0 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 ) diff --git a/go.sum b/go.sum index 7760462..86e56c0 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,6 @@ github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= -github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= From 00764f065631fa5a903476f57f157a50cdddd91c Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 19 Nov 2020 19:24:39 -0800 Subject: [PATCH 22/29] Encode using multibase --- dsindex/indexer.go | 12 +++++---- dspinner/pin.go | 4 +-- dspinner/pin_test.go | 43 +++++++++++++++++++++++++++++ go.mod | 15 ++++++----- go.sum | 64 +++++++++++++++++++++++++++++++++++--------- 5 files changed, 112 insertions(+), 26 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 485d59b..460f5a9 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -3,12 +3,12 @@ package dsindex import ( "context" - "encoding/base64" "fmt" "path" ds "github.com/ipfs/go-datastore" query "github.com/ipfs/go-datastore/query" + "github.com/multiformats/go-multibase" ) // Indexer maintains a secondary index. Each value of the secondary index maps @@ -264,14 +264,16 @@ func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry } func encode(data string) string { - if data == "" { - return "" + encData, err := multibase.Encode(multibase.Base64url, []byte(data)) + if err != nil { + // programming error; using unsupported encoding + panic(err.Error()) } - return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString([]byte(data)) + return encData } func decode(data string) (string, error) { - b, err := base64.URLEncoding.WithPadding(base64.NoPadding).DecodeString(data) + _, b, err := multibase.Decode(data) if err != nil { return "", err } diff --git a/dspinner/pin.go b/dspinner/pin.go index 4dec4b4..a7988d2 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -131,7 +131,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { // temporary unlock to fetch the entire graph p.lock.Unlock() - // Fetch graph starting a node identified by cid + // Fetch graph starting at node identified by cid err = mdag.FetchGraph(ctx, c, p.dserv) p.lock.Lock() if err != nil { @@ -938,7 +938,7 @@ func decodePin(pid string, data []byte) (*pin, error) { if !ok { return nil, fmt.Errorf("missing mode") } - mode64, ok := modeData.(uint64) + mode64, ok := modeData.(int) if !ok { return nil, fmt.Errorf("invalid pin mode data") } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 02799c1..1090b10 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -659,6 +659,49 @@ func TestLoadDirty(t *testing.T) { } } +func TestEncodeDecodePin(t *testing.T) { + _, c := randNode() + + pin := newPin(c, ipfspin.Recursive, "testpin") + pin.metadata = make(map[string]interface{}, 2) + pin.metadata["hello"] = "world" + pin.metadata["foo"] = "bar" + + encBytes, err := encodePin(pin) + if err != nil { + t.Fatal(err) + } + + decPin, err := decodePin(pin.id, encBytes) + if err != nil { + t.Fatal(err) + } + + if decPin.id != pin.id { + t.Errorf("wrong pin id: expect %q got %q", pin.id, decPin.id) + } + if decPin.cid != pin.cid { + t.Errorf("wrong pin cid: expect %q got %q", pin.cid.String(), decPin.cid.String()) + } + if decPin.mode != pin.mode { + expect, _ := ipfspin.ModeToString(pin.mode) + got, _ := ipfspin.ModeToString(decPin.mode) + t.Errorf("wrong pin mode: expect %s got %s", expect, got) + } + if decPin.name != pin.name { + t.Errorf("wrong pin name: expect %q got %q", pin.name, decPin.name) + } + for key, val := range pin.metadata { + dval, ok := decPin.metadata[key] + if !ok { + t.Errorf("decoded pin missing metadata key %q", key) + } + if dval != val { + t.Errorf("wrong metadata value: expected %q got %q", val, dval) + } + } +} + func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) { if aBranchLen < 3 { err = errors.New("set aBranchLen to at least 3") diff --git a/go.mod b/go.mod index 30a08c9..5fb384a 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,16 @@ go 1.13 require ( github.com/gogo/protobuf v1.3.1 - github.com/ipfs/go-blockservice v0.1.2 - github.com/ipfs/go-cid v0.0.3 + github.com/ipfs/go-blockservice v0.1.4 + github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-datastore v0.4.5 github.com/ipfs/go-ds-leveldb v0.4.2 - github.com/ipfs/go-ipfs-blockstore v0.1.0 + github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 - github.com/ipfs/go-ipfs-util v0.0.1 - github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v0.0.1 + github.com/ipfs/go-ipfs-util v0.0.2 + github.com/ipfs/go-ipld-format v0.2.0 + github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.0 - github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 + github.com/multiformats/go-multibase v0.0.3 + github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 ) diff --git a/go.sum b/go.sum index 86e56c0..fc7de89 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= @@ -51,6 +52,8 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= @@ -61,25 +64,27 @@ github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0r github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= -github.com/ipfs/go-bitswap v0.1.3 h1:jAl9Z/TYObpGeGATUemnOZ7RYb0F/kzNVlhcYZesz+0= -github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= +github.com/ipfs/go-bitswap v0.1.8 h1:38X1mKXkiU6Nzw4TOSWD8eTVY5eX3slQunv3QEWfXKg= +github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-blockservice v0.1.0 h1:dh2i7xjMbCtf0ZSMyQAF2qpV/pEEmM7yVpQ00+gik6U= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= -github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= -github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= +github.com/ipfs/go-blockservice v0.1.4 h1:Vq+MlsH8000KbbUciRyYMEw/NNP8UAGmcqKi4uWmFGA= +github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= -github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= -github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.1 h1:W4ZfzyhNi3xmuU5dQhjfuRn/wFuqEE1KnOmmQiOevEY= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -91,8 +96,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= -github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= -github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= +github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= +github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -100,6 +105,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.1 h1:IW/bXGeaAZV2VH0Kuok+Ohva/zHkHmeLFBxC1k7mNPc= +github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= @@ -110,13 +117,21 @@ github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRD github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.0.2 h1:amzFztBQQQ69UA5+f7JRfoXF/z2l//MGfEDHVkS20+s= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= +github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= +github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -216,8 +231,8 @@ github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTW github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= -github.com/libp2p/go-msgio v0.0.3 h1:VsOlWispTivSsOMg70e0W77y6oiSBSRCyP6URrWvE04= -github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= +github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= @@ -249,12 +264,18 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= @@ -268,11 +289,18 @@ github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5 h1:1wxmCvTXAifAepIMyF39vZinRw5sbqjPs/UIi93+uik= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -289,6 +317,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -303,6 +333,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= @@ -318,9 +350,14 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -332,6 +369,7 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -365,6 +403,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -382,4 +421,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From a6d812c3e9b10d8a3c1cffd1c7c22a0faf5bb498 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 23 Nov 2020 19:31:11 -0800 Subject: [PATCH 23/29] Changes from review - Change naming from "index" and "key" to "key" and "value" - Use wrapped datastore instead of using index key directly - Fix typo in comment --- dsindex/error.go | 2 +- dsindex/indexer.go | 169 ++++++++++++++++++++-------------------- dsindex/indexer_test.go | 83 +++++++++++--------- dspinner/pin.go | 24 +++--- dspinner/pin_test.go | 47 +++++++---- 5 files changed, 174 insertions(+), 151 deletions(-) diff --git a/dsindex/error.go b/dsindex/error.go index b61d90d..f3b685b 100644 --- a/dsindex/error.go +++ b/dsindex/error.go @@ -3,6 +3,6 @@ package dsindex import "errors" var ( - ErrEmptyIndex = errors.New("index is empty") ErrEmptyKey = errors.New("key is empty") + ErrEmptyValue = errors.New("value is empty") ) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 460f5a9..2688d69 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -7,103 +7,105 @@ import ( "path" ds "github.com/ipfs/go-datastore" - query "github.com/ipfs/go-datastore/query" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" "github.com/multiformats/go-multibase" ) -// Indexer maintains a secondary index. Each value of the secondary index maps -// to one more primary keys. +// Indexer maintains a secondary index. An index is a collection of key-value +// mappings where the key is the secondary index that maps to one or more +// values, where each value is a unique key being indexed. type Indexer interface { - // Add adds the specified key to the an index - Add(ctx context.Context, index, key string) error + // Add adds the specified value to the key + Add(ctx context.Context, key, value string) error - // Delete deletes the specified key from the index. If the key is not in + // Delete deletes the specified value from the key. If the value is not in // the datastore, this method returns no error. - Delete(ctx context.Context, index, key string) error + Delete(ctx context.Context, key, value string) error - // DeleteIndex deletes all keys in the given index. If an index is not in - // the datastore, this method returns no error. - DeleteIndex(ctx context.Context, index string) (count int, err error) + // DeleteKey deletes all values in the given key. If a key is not in the + // datastore, this method returns no error. Returns a count of values that + // were deleted. + DeleteKey(ctx context.Context, key string) (count int, err error) - // DeleteAll deletes all indexes managed by this Indexer + // DeleteAll deletes all keys managed by this Indexer. Returns a count of + // the values that were deleted. DeleteAll(ctx context.Context) (count int, err error) - // ForEach calls the function for each key in the specified index, until - // there are no more keys, or until the function returns false. If index - // is empty string, then all indexs are iterated. - ForEach(ctx context.Context, index string, fn func(index, key string) bool) error + // ForEach calls the function for each value in the specified index, until + // there are no more values, or until the function returns false. If key + // is empty string, then all keys are iterated. + ForEach(ctx context.Context, index string, fn func(key, value string) bool) error - // HasKey determines if the index contains the specified key - HasKey(ctx context.Context, index, key string) (bool, error) + // HasValue determines if the key contains the specified value + HasValue(ctx context.Context, key, value string) (bool, error) - // HasAny determines if any key is in the specified index. If index is - // empty string, then all indexes are searched. - HasAny(ctx context.Context, index string) (bool, error) + // HasAny determines if any value is in the specified key. If key is + // empty string, then all values are searched. + HasAny(ctx context.Context, key string) (bool, error) - // Search returns all keys for the given index - Search(ctx context.Context, index string) (ids []string, err error) + // Search returns all values for the given key + Search(ctx context.Context, key string) (values []string, err error) } // indexer is a simple implementation of Indexer. This implementation relies -// on the underlying data store supporting efficent querying by prefix. +// on the underlying data store to support efficient querying by prefix. // // TODO: Consider adding caching type indexer struct { - dstore ds.Datastore - indexPath string + dstore ds.Datastore } -// New creates a new datastore index. All indexes are stored prefixed with the -// specified index path. +// New creates a new datastore index. All indexes are stored under the +// specified index name. // // To persist the actions of calling Indexer functions, it is necessary to call // dstore.Sync. -func New(dstore ds.Datastore, indexPath string) Indexer { +func New(dstore ds.Datastore, name string) Indexer { return &indexer{ - dstore: dstore, - indexPath: indexPath, + dstore: namespace.Wrap(dstore, ds.NewKey(name)), } } -func (x *indexer) Add(ctx context.Context, index, key string) error { - if index == "" { - return ErrEmptyIndex - } +func (x *indexer) Add(ctx context.Context, key, value string) error { if key == "" { return ErrEmptyKey } - dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) - return x.dstore.Put(dskey, []byte{}) + if value == "" { + return ErrEmptyValue + } + dsKey := ds.NewKey(encode(key)).ChildString(encode(value)) + return x.dstore.Put(dsKey, []byte{}) } -func (x *indexer) Delete(ctx context.Context, index, key string) error { - if index == "" { - return ErrEmptyIndex - } +func (x *indexer) Delete(ctx context.Context, key, value string) error { if key == "" { return ErrEmptyKey } - dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) - return x.dstore.Delete(dskey) + if value == "" { + return ErrEmptyValue + } + return x.dstore.Delete(ds.NewKey(encode(key)).ChildString(encode(value))) } -func (x *indexer) DeleteIndex(ctx context.Context, index string) (int, error) { - if index == "" { - return 0, ErrEmptyIndex +func (x *indexer) DeleteKey(ctx context.Context, key string) (int, error) { + if key == "" { + return 0, ErrEmptyKey } - return x.deletePrefix(ctx, path.Join(x.indexPath, encode(index))) + return x.deletePrefix(ctx, encode(key)) } func (x *indexer) DeleteAll(ctx context.Context) (int, error) { - return x.deletePrefix(ctx, x.indexPath) + return x.deletePrefix(ctx, "") } -func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, key string) bool) error { - if index != "" { - index = encode(index) +func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value string) bool) error { + if key != "" { + key = encode(key) } + q := query.Query{ - Prefix: path.Join(x.indexPath, index), + Prefix: key, KeysOnly: true, } results, err := x.dstore.Query(q) @@ -141,31 +143,30 @@ func (x *indexer) ForEach(ctx context.Context, index string, fn func(idx, key st return err } -func (x *indexer) HasKey(ctx context.Context, index, key string) (bool, error) { - if index == "" { - return false, ErrEmptyIndex - } +func (x *indexer) HasValue(ctx context.Context, key, value string) (bool, error) { if key == "" { return false, ErrEmptyKey } - dskey := ds.NewKey(path.Join(x.indexPath, encode(index), encode(key))) - return x.dstore.Has(dskey) + if value == "" { + return false, ErrEmptyValue + } + return x.dstore.Has(ds.NewKey(encode(key)).ChildString(encode(value))) } -func (x *indexer) HasAny(ctx context.Context, index string) (bool, error) { +func (x *indexer) HasAny(ctx context.Context, key string) (bool, error) { var any bool - err := x.ForEach(ctx, index, func(idx, key string) bool { + err := x.ForEach(ctx, key, func(key, value string) bool { any = true return false }) return any, err } -func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { - if index == "" { - return nil, ErrEmptyIndex +func (x *indexer) Search(ctx context.Context, key string) ([]string, error) { + if key == "" { + return nil, ErrEmptyKey } - ents, err := x.queryPrefix(ctx, path.Join(x.indexPath, encode(index))) + ents, err := x.queryPrefix(ctx, encode(key)) if err != nil { return nil, err } @@ -173,24 +174,24 @@ func (x *indexer) Search(ctx context.Context, index string) ([]string, error) { return nil, nil } - keys := make([]string, len(ents)) + values := make([]string, len(ents)) for i := range ents { - keys[i], err = decode(path.Base(ents[i].Key)) + values[i], err = decode(path.Base(ents[i].Key)) if err != nil { - return nil, fmt.Errorf("cannot decode key: %v", err) + return nil, fmt.Errorf("cannot decode value: %v", err) } } - return keys, nil + return values, nil } -// SyncIndex synchronizes the indexes in the target Indexer to match those of -// the ref Indexer. The indexPath prefix is not synchronized, only the -// index/key portion of the indexes. +// SyncIndex synchronizes the keys in the target Indexer to match those of the +// ref Indexer. The name portion of the stored data is not synchronized, only +// the key/value portion of the indexes. func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Build reference index map refs := map[string]string{} - err := ref.ForEach(ctx, "", func(idx, key string) bool { - refs[key] = idx + err := ref.ForEach(ctx, "", func(key, value string) bool { + refs[value] = key return true }) if err != nil { @@ -202,13 +203,13 @@ func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Compare current indexes dels := map[string]string{} - err = target.ForEach(ctx, "", func(idx, key string) bool { - refIdx, ok := refs[key] - if ok && refIdx == idx { - // same in both; delete from refs, do not add to delKeys - delete(refs, key) + err = target.ForEach(ctx, "", func(key, value string) bool { + refKey, ok := refs[value] + if ok && refKey == key { + // same in both; delete from refs, do not add to dels + delete(refs, value) } else { - dels[key] = idx + dels[value] = key } return true }) @@ -216,17 +217,17 @@ func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { return false, err } - // Items in dels are indexes that no longer exist - for key, idx := range dels { - err = target.Delete(ctx, idx, key) + // Items in dels are keys that no longer exist + for value, key := range dels { + err = target.Delete(ctx, key, value) if err != nil { return false, err } } - // What remains in refs are indexes that need to be added - for key, idx := range refs { - err = target.Add(ctx, idx, key) + // What remains in refs are keys that need to be added + for value, key := range refs { + err = target.Add(ctx, key, value) if err != nil { return false, err } diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go index 6c2f781..dcea2c1 100644 --- a/dsindex/indexer_test.go +++ b/dsindex/indexer_test.go @@ -21,7 +21,8 @@ func createIndexer() Indexer { } func TestAdd(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() err := nameIndex.Add(ctx, "someone", "s1") if err != nil { @@ -33,21 +34,22 @@ func TestAdd(t *testing.T) { } err = nameIndex.Add(ctx, "", "noindex") - if err != ErrEmptyIndex { + if err != ErrEmptyKey { t.Fatal("unexpected error:", err) } err = nameIndex.Add(ctx, "nokey", "") - if err != ErrEmptyKey { + if err != ErrEmptyValue { t.Fatal("unexpected error:", err) } } -func TestHasKey(t *testing.T) { - ctx := context.Background() +func TestHasValue(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() - ok, err := nameIndex.HasKey(ctx, "bob", "b1") + ok, err := nameIndex.HasValue(ctx, "bob", "b1") if err != nil { t.Fatal(err) } @@ -55,7 +57,7 @@ func TestHasKey(t *testing.T) { t.Fatal("missing index") } - ok, err = nameIndex.HasKey(ctx, "bob", "b3") + ok, err = nameIndex.HasValue(ctx, "bob", "b3") if err != nil { t.Fatal(err) } @@ -63,19 +65,20 @@ func TestHasKey(t *testing.T) { t.Fatal("should not have index") } - _, err = nameIndex.HasKey(ctx, "", "b1") - if err != ErrEmptyIndex { + _, err = nameIndex.HasValue(ctx, "", "b1") + if err != ErrEmptyKey { t.Fatal("unexpected error:", err) } - _, err = nameIndex.HasKey(ctx, "bob", "") - if err != ErrEmptyKey { + _, err = nameIndex.HasValue(ctx, "bob", "") + if err != ErrEmptyValue { t.Fatal("unexpected error:", err) } } func TestHasAny(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() ok, err := nameIndex.HasAny(ctx, "nothere") @@ -114,53 +117,55 @@ func TestHasAny(t *testing.T) { } func TestForEach(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() found := make(map[string]struct{}) - err := nameIndex.ForEach(ctx, "bob", func(idx, id string) bool { - found[id] = struct{}{} + err := nameIndex.ForEach(ctx, "bob", func(key, value string) bool { + found[value] = struct{}{} return true }) if err != nil { t.Fatal(err) } - for _, idx := range []string{"b1", "b2"} { - _, ok := found[idx] + for _, value := range []string{"b1", "b2"} { + _, ok := found[value] if !ok { - t.Fatal("missing index for key", idx) + t.Fatal("missing key for value", value) } } - keys := map[string]string{} - err = nameIndex.ForEach(ctx, "", func(idx, id string) bool { - keys[id] = idx + values := map[string]string{} + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + values[value] = key return true }) if err != nil { t.Fatal(err) } - if len(keys) != 4 { + if len(values) != 4 { t.Fatal("expected 4 keys") } - if keys["a1"] != "alice" { + if values["a1"] != "alice" { t.Error("expected a1: alice") } - if keys["b1"] != "bob" { + if values["b1"] != "bob" { t.Error("expected b1: bob") } - if keys["b2"] != "bob" { + if values["b2"] != "bob" { t.Error("expected b2: bob") } - if keys["c1"] != "cathy" { + if values["c1"] != "cathy" { t.Error("expected c1: cathy") } } func TestSearch(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() ids, err := nameIndex.Search(ctx, "bob") @@ -197,7 +202,8 @@ func TestSearch(t *testing.T) { } func TestDelete(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() err := nameIndex.Delete(ctx, "bob", "b3") @@ -210,7 +216,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } - ok, err := nameIndex.HasKey(ctx, "alice", "a1") + ok, err := nameIndex.HasValue(ctx, "alice", "a1") if err != nil { t.Fatal(err) } @@ -218,21 +224,22 @@ func TestDelete(t *testing.T) { t.Fatal("index key should have been deleted") } - count, err := nameIndex.DeleteIndex(ctx, "bob") + count, err := nameIndex.DeleteKey(ctx, "bob") if err != nil { t.Fatal(err) } if count != 2 { t.Fatal("wrong deleted count") } - ok, _ = nameIndex.HasKey(ctx, "bob", "b1") + ok, _ = nameIndex.HasValue(ctx, "bob", "b1") if ok { t.Fatal("index not deleted") } } func TestSyncIndex(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nameIndex := createIndexer() dstore := ds.NewMapDatastore() @@ -251,8 +258,8 @@ func TestSyncIndex(t *testing.T) { // Create map of id->index in sync target syncs := map[string]string{} - err = nameIndex.ForEach(ctx, "", func(idx, id string) bool { - syncs[id] = idx + err = nameIndex.ForEach(ctx, "", func(key, value string) bool { + syncs[value] = key return true }) if err != nil { @@ -261,11 +268,11 @@ func TestSyncIndex(t *testing.T) { // Iterate items in sync source and make sure they appear in target var itemCount int - err = refIndex.ForEach(ctx, "", func(idx, id string) bool { + err = refIndex.ForEach(ctx, "", func(key, value string) bool { itemCount++ - syncIdx, ok := syncs[id] - if !ok || idx != syncIdx { - t.Fatal("index", idx, "-->", id, "was not synced") + syncKey, ok := syncs[value] + if !ok || key != syncKey { + t.Fatal("key", key, "-->", value, "was not synced") } return true }) diff --git a/dspinner/pin.go b/dspinner/pin.go index a7988d2..f7dd3d3 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -370,8 +370,8 @@ func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinne var has bool var rc cid.Cid var e error - err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { - rc, e = cid.Cast([]byte(idx)) + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { + rc, e = cid.Cast([]byte(key)) if e != nil { return false } @@ -460,9 +460,9 @@ func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinn } var e error - err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { var rk cid.Cid - rk, e = cid.Cast([]byte(idx)) + rk, e = cid.Cast([]byte(key)) if e != nil { return false } @@ -551,12 +551,12 @@ func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinne // Fix index; remove index for pin that does not exist switch mode { case ipfspinner.Recursive: - p.cidRIndex.DeleteIndex(ctx, cidKey) + p.cidRIndex.DeleteKey(ctx, cidKey) case ipfspinner.Direct: - p.cidDIndex.DeleteIndex(ctx, cidKey) + p.cidDIndex.DeleteKey(ctx, cidKey) case ipfspinner.Any: - p.cidRIndex.DeleteIndex(ctx, cidKey) - p.cidDIndex.DeleteIndex(ctx, cidKey) + p.cidRIndex.DeleteKey(ctx, cidKey) + p.cidDIndex.DeleteKey(ctx, cidKey) } log.Error("found CID index with missing pin") continue @@ -707,9 +707,9 @@ func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { cidSet := cid.NewSet() var e error - err := p.cidDIndex.ForEach(ctx, "", func(idx, id string) bool { + err := p.cidDIndex.ForEach(ctx, "", func(key, value string) bool { var c cid.Cid - c, e = cid.Cast([]byte(idx)) + c, e = cid.Cast([]byte(key)) if e != nil { return false } @@ -733,9 +733,9 @@ func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) { cidSet := cid.NewSet() var e error - err := p.cidRIndex.ForEach(ctx, "", func(idx, id string) bool { + err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool { var c cid.Cid - c, e = cid.Cast([]byte(idx)) + c, e = cid.Cast([]byte(key)) if e != nil { return false } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index 1090b10..c6296f0 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -97,7 +97,8 @@ func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) { } func TestPinnerBasic(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) @@ -316,7 +317,8 @@ func TestPinnerBasic(t *testing.T) { } func TestRemovePinWithMode(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) @@ -361,7 +363,8 @@ func TestIsPinnedLookup(t *testing.T) { // are pinned and once they have been unpinned. aBranchLen := 6 - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) @@ -403,7 +406,8 @@ func TestIsPinnedLookup(t *testing.T) { } func TestDuplicateSemantics(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) @@ -441,7 +445,8 @@ func TestDuplicateSemantics(t *testing.T) { } func TestFlush(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) @@ -461,7 +466,8 @@ func TestFlush(t *testing.T) { } func TestPinRecursiveFail(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) bserv := bs.New(bstore, offline.Exchange(bstore)) @@ -508,7 +514,8 @@ func TestPinRecursiveFail(t *testing.T) { } func TestPinUpdate(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) @@ -575,7 +582,8 @@ func TestPinUpdate(t *testing.T) { } func TestLoadDirty(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore := dssync.MutexWrap(ds.NewMapDatastore()) bstore := blockstore.NewBlockstore(dstore) @@ -605,7 +613,7 @@ func TestLoadDirty(t *testing.T) { // Corrupt index cidRIndex := p.(*pinner).cidRIndex - cidRIndex.DeleteIndex(ctx, cidAKey) + cidRIndex.DeleteKey(ctx, cidAKey) cidRIndex.Add(ctx, cidBKey, "not-a-pin-id") // Verify dirty @@ -776,7 +784,8 @@ func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfs } func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nodes := make([]ipld.Node, count) for i := 0; i < count; i++ { n, _ := randNode() @@ -790,7 +799,8 @@ func makeNodes(count int, dserv ipld.DAGService) []ipld.Node { } func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var err error for i := range nodes { @@ -806,7 +816,8 @@ func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) { } func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() var err error for i := range nodes { @@ -847,7 +858,8 @@ func makeStore() (ds.Datastore, ipld.DAGService) { // compares the load time when rebuilding indexes to loading without rebuilding // indexes. func BenchmarkLoadRebuild(b *testing.B) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() dstore, dserv := makeStore() pinner, err := LoadPinner(ctx, dstore, dserv) @@ -904,7 +916,8 @@ func BenchmarkNthPin(b *testing.B) { } func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nodes := makeNodes(count, dserv) pinNodes(nodes[:count-1], pinner, true) b.ResetTimer() @@ -956,7 +969,8 @@ func BenchmarkNPins(b *testing.B) { } func benchmarkNPins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nodes := makeNodes(count, dserv) b.ResetTimer() @@ -1002,7 +1016,8 @@ func BenchmarkNUnpins(b *testing.B) { } func benchmarkNUnpins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() nodes := makeNodes(count, dserv) pinNodes(nodes, pinner, true) b.ResetTimer() From 86f36c2a1874102a9e4c530faa5d7d177802ee2f Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 23 Nov 2020 22:34:32 -0800 Subject: [PATCH 24/29] Rename LoadPinner to New for both pinners --- dspinner/pin.go | 69 ++++++++++++----------- dspinner/pin_test.go | 56 +++++++++++-------- ipldpinner/pin.go | 121 ++++++++++++++++++++-------------------- ipldpinner/pin_test.go | 41 ++++++++++---- pinconv/pinconv.go | 8 +-- pinconv/pinconv_test.go | 2 +- 6 files changed, 163 insertions(+), 134 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index f7dd3d3..dcc30fe 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -105,6 +105,41 @@ type syncDAGService interface { Sync() error } +// New creates a new pinner and loads its keysets from the given datastore. If +// there is no data present in the datastore, then an empty pinner is returned. +func New(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { + p := &pinner{ + cidDIndex: dsindex.New(dstore, pinCidDIndexPath), + cidRIndex: dsindex.New(dstore, pinCidRIndexPath), + nameIndex: dsindex.New(dstore, pinNameIndexPath), + dserv: dserv, + dstore: dstore, + } + + data, err := dstore.Get(dirtyKey) + if err != nil { + if err == ds.ErrNotFound { + return p, nil + } + return nil, fmt.Errorf("cannot load dirty flag: %v", err) + } + if data[0] == 1 { + p.dirty = 1 + + pins, err := p.loadAllPins(ctx) + if err != nil { + return nil, fmt.Errorf("cannot load pins: %v", err) + } + + err = p.rebuildIndexes(ctx, pins) + if err != nil { + return nil, fmt.Errorf("cannot rebuild indexes: %v", err) + } + } + + return p, nil +} + // Pin the given node, optionally recursive func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { err := p.dserv.Add(ctx, node) @@ -612,40 +647,6 @@ func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { return pins, nil } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { - p := &pinner{ - cidDIndex: dsindex.New(dstore, pinCidDIndexPath), - cidRIndex: dsindex.New(dstore, pinCidRIndexPath), - nameIndex: dsindex.New(dstore, pinNameIndexPath), - dserv: dserv, - dstore: dstore, - } - - data, err := dstore.Get(dirtyKey) - if err != nil { - if err == ds.ErrNotFound { - return p, nil - } - return nil, fmt.Errorf("cannot load dirty flag: %v", err) - } - if data[0] == 1 { - p.dirty = 1 - - pins, err := p.loadAllPins(ctx) - if err != nil { - return nil, fmt.Errorf("cannot load pins: %v", err) - } - - err = p.rebuildIndexes(ctx, pins) - if err != nil { - return nil, fmt.Errorf("cannot rebuild indexes: %v", err) - } - } - - return p, nil -} - // rebuildIndexes uses the stored pins to rebuild secondary indexes. This // resolves any discrepancy between secondary indexes and pins that could // result from a program termination between saving the two. diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index c6296f0..c7bd766 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -106,7 +106,7 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -270,7 +270,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - p, err = LoadPinner(ctx, dstore, dserv) + p, err = New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -326,7 +326,7 @@ func TestRemovePinWithMode(t *testing.T) { dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -371,9 +371,9 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) - // Create new pinner. LoadPinner will not load anything since there are + // Create new pinner. New will not load anything since there are // no pins saved in the datastore yet. - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -414,7 +414,7 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -452,7 +452,7 @@ func TestFlush(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -473,7 +473,7 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -522,7 +522,7 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -590,7 +590,7 @@ func TestLoadDirty(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p, err := LoadPinner(ctx, dstore, dserv) + p, err := New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -634,7 +634,7 @@ func TestLoadDirty(t *testing.T) { } // Create new pinner on same datastore that was never flushed. - p, err = LoadPinner(ctx, dstore, dserv) + p, err = New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } @@ -862,7 +862,7 @@ func BenchmarkLoadRebuild(b *testing.B) { defer cancel() dstore, dserv := makeStore() - pinner, err := LoadPinner(ctx, dstore, dserv) + pinner, err := New(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -874,7 +874,7 @@ func BenchmarkLoadRebuild(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{1}) - _, err = LoadPinner(ctx, dstore, dserv) + _, err = New(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -885,7 +885,7 @@ func BenchmarkLoadRebuild(b *testing.B) { for i := 0; i < b.N; i++ { dstore.Put(dirtyKey, []byte{0}) - _, err = LoadPinner(ctx, dstore, dserv) + _, err = New(ctx, dstore, dserv) if err != nil { panic(err.Error()) } @@ -898,11 +898,14 @@ func BenchmarkLoadRebuild(b *testing.B) { // creating a pin in a larger number of existing pins. func BenchmarkNthPin(b *testing.B) { dstore, dserv := makeStore() - pinner, err := LoadPinner(context.Background(), dstore, dserv) + pinner, err := New(context.Background(), dstore, dserv) + if err != nil { + panic(err.Error()) + } + pinnerIPLD, err := ipldpinner.New(dstore, dserv, dserv) if err != nil { panic(err.Error()) } - pinnerIPLD := ipldpinner.New(dstore, dserv, dserv) for count := 1000; count <= 10000; count += 1000 { b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { @@ -953,7 +956,7 @@ func BenchmarkNPins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner, err := LoadPinner(context.Background(), dstore, dserv) + pinner, err := New(context.Background(), dstore, dserv) if err != nil { panic(err.Error()) } @@ -962,7 +965,10 @@ func BenchmarkNPins(b *testing.B) { b.Run(fmt.Sprint("PinIPLD-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } benchmarkNPins(b, count, pinner, dserv) }) } @@ -1000,7 +1006,7 @@ func BenchmarkNUnpins(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner, err := LoadPinner(context.Background(), dstore, dserv) + pinner, err := New(context.Background(), dstore, dserv) if err != nil { panic(err.Error()) } @@ -1009,7 +1015,10 @@ func BenchmarkNUnpins(b *testing.B) { b.Run(fmt.Sprint("UninIPLD-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } benchmarkNUnpins(b, count, pinner, dserv) }) } @@ -1047,7 +1056,7 @@ func BenchmarkPinAll(b *testing.B) { for count := 128; count < 16386; count <<= 1 { b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner, err := LoadPinner(context.Background(), dstore, dserv) + pinner, err := New(context.Background(), dstore, dserv) if err != nil { panic(err) } @@ -1056,7 +1065,10 @@ func BenchmarkPinAll(b *testing.B) { b.Run(fmt.Sprint("PinAllIPLD-", count), func(b *testing.B) { dstore, dserv := makeStore() - pinner := ipldpinner.New(dstore, dserv, dserv) + pinner, err := ipldpinner.New(dstore, dserv, dserv) + if err != nil { + panic(err.Error()) + } benchmarkPinAll(b, count, pinner, dserv) }) } diff --git a/ipldpinner/pin.go b/ipldpinner/pin.go index 742bd2b..d0824b3 100644 --- a/ipldpinner/pin.go +++ b/ipldpinner/pin.go @@ -78,16 +78,67 @@ type syncDAGService interface { Sync() error } -// New creates a new pinner using the given datastore as a backend -func New(dstore ds.Datastore, serv, internal ipld.DAGService) *pinner { - return &pinner{ - recursePin: cid.NewSet(), - directPin: cid.NewSet(), - internalPin: cid.NewSet(), - dserv: serv, - internal: internal, - dstore: dstore, +// New creates a new pinner using the given datastore as a backend, and loads +// the pinner's keysets from the datastore +func New(dstore ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { + rootKey, err := dstore.Get(pinDatastoreKey) + if err != nil { + if err == ds.ErrNotFound { + return &pinner{ + recursePin: cid.NewSet(), + directPin: cid.NewSet(), + internalPin: cid.NewSet(), + dserv: dserv, + internal: internal, + dstore: dstore, + }, nil + } + return nil, err } + rootCid, err := cid.Cast(rootKey) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) + defer cancel() + + root, err := internal.Get(ctx, rootCid) + if err != nil { + return nil, fmt.Errorf("cannot find pinning root object: %v", err) + } + + rootpb, ok := root.(*mdag.ProtoNode) + if !ok { + return nil, mdag.ErrNotProtobuf + } + + internalset := cid.NewSet() + internalset.Add(rootCid) + recordInternal := internalset.Add + + // load recursive set + recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load recursive pins: %v", err) + } + + // load direct set + directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) + if err != nil { + return nil, fmt.Errorf("cannot load direct pins: %v", err) + } + + return &pinner{ + // assign pinsets + recursePin: cidSetWithValues(recurseKeys), + directPin: cidSetWithValues(directKeys), + internalPin: internalset, + // assign services + dserv: dserv, + dstore: dstore, + internal: internal, + }, nil } // Pin the given node, optionally recursive @@ -312,58 +363,6 @@ func cidSetWithValues(cids []cid.Cid) *cid.Set { return out } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(dstore ds.Datastore, dserv, internal ipld.DAGService) (*pinner, error) { - rootKey, err := dstore.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(context.TODO(), loadTimeout) - defer cancel() - - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add - - // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) - } - - // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - - return &pinner{ - // assign pinsets - recursePin: cidSetWithValues(recurseKeys), - directPin: cidSetWithValues(directKeys), - internalPin: internalset, - // assign services - dserv: dserv, - dstore: dstore, - internal: internal, - }, nil -} - // DirectKeys returns a slice containing the directly pinned keys func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) { p.lock.RLock() diff --git a/ipldpinner/pin_test.go b/ipldpinner/pin_test.go index d2fbfb5..e193aa9 100644 --- a/ipldpinner/pin_test.go +++ b/ipldpinner/pin_test.go @@ -63,10 +63,13 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, ak := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -152,7 +155,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv, dserv) + np, err := New(dstore, dserv, dserv) if err != nil { t.Fatal(err) } @@ -189,7 +192,10 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } aNodes := make([]*mdag.ProtoNode, aBranchLen) aKeys := make([]cid.Cid, aBranchLen) @@ -230,7 +236,7 @@ func TestIsPinnedLookup(t *testing.T) { } // Add C - err := dserv.Add(ctx, c) + err = dserv.Add(ctx, c) if err != nil { t.Fatal(err) } @@ -290,11 +296,13 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) - // TODO does pinner need to share datastore with blockservice? - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() - err := dserv.Add(ctx, a) + err = dserv.Add(ctx, a) if err != nil { t.Fatal(err) } @@ -324,7 +332,10 @@ func TestFlush(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } _, k := randNode() p.PinWithMode(k, pin.Recursive) @@ -341,11 +352,14 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } a, _ := randNode() b, _ := randNode() - err := a.AddNodeLink("child", b) + err = a.AddNodeLink("child", b) if err != nil { t.Fatal(err) } @@ -386,7 +400,10 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := New(dstore, dserv, dserv) + p, err := New(dstore, dserv, dserv) + if err != nil { + t.Fatal(err) + } n1, c1 := randNode() n2, c2 := randNode() diff --git a/pinconv/pinconv.go b/pinconv/pinconv.go index 0c1c093..9aee703 100644 --- a/pinconv/pinconv.go +++ b/pinconv/pinconv.go @@ -24,12 +24,12 @@ import ( func ConvertPinsFromIPLDToDS(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { const ipldPinPath = "/local/pins" - ipldPinner, err := ipldpinner.LoadPinner(dstore, dserv, internal) + ipldPinner, err := ipldpinner.New(dstore, dserv, internal) if err != nil { return nil, 0, err } - dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + dsPinner, err := dspinner.New(ctx, dstore, dserv) if err != nil { return nil, 0, err } @@ -83,12 +83,12 @@ func ConvertPinsFromIPLDToDS(ctx context.Context, dstore ds.Datastore, dserv ipl // After the pins are stored in the DAGService, the pins and their indexes are // removed from the dspinner. func ConvertPinsFromDSToIPLD(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService, internal ipld.DAGService) (ipfspinner.Pinner, int, error) { - dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + dsPinner, err := dspinner.New(ctx, dstore, dserv) if err != nil { return nil, 0, err } - ipldPinner := ipldpinner.New(dstore, dserv, internal) + ipldPinner, err := ipldpinner.New(dstore, dserv, internal) if err != nil { return nil, 0, err } diff --git a/pinconv/pinconv_test.go b/pinconv/pinconv_test.go index 345c6a1..ac7f8ff 100644 --- a/pinconv/pinconv_test.go +++ b/pinconv/pinconv_test.go @@ -58,7 +58,7 @@ func TestConversions(t *testing.T) { ctx := context.Background() dstore, dserv := makeStore() - dsPinner, err := dspinner.LoadPinner(ctx, dstore, dserv) + dsPinner, err := dspinner.New(ctx, dstore, dserv) if err != nil { t.Fatal(err) } From 7a128c682069803ae0a9a18d3c7c5d31441bda2b Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 24 Nov 2020 09:11:27 -0800 Subject: [PATCH 25/29] Check context when loading pinner and during iterative operations --- dsindex/indexer.go | 5 ++++- dspinner/pin.go | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 2688d69..35a5e0f 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -122,7 +122,10 @@ func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value st err = r.Error break } - + if ctx.Err() != nil { + err = ctx.Err() + break + } ent := r.Entry decIdx, err := decode(path.Base(path.Dir(ent.Key))) if err != nil { diff --git a/dspinner/pin.go b/dspinner/pin.go index dcc30fe..d96f9e2 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -637,6 +637,9 @@ func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { pins := make([]*pin, len(ents)) for i := range ents { + if ctx.Err() != nil { + return nil, ctx.Err() + } var p *pin p, err = decodePin(path.Base(ents[i].Key), ents[i].Value) if err != nil { @@ -658,6 +661,9 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { tmpNameIndex := dsindex.New(dstoreMem, pinNameIndexPath) var hasNames bool for _, pp := range pins { + if ctx.Err() != nil { + return ctx.Err() + } if pp.mode == ipfspinner.Recursive { tmpCidRIndex.Add(ctx, pp.cid.KeyString(), pp.id) } else if pp.mode == ipfspinner.Direct { From cd2065d029b249a6fe8e363eea0c1ac54fb3df02 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 24 Nov 2020 09:47:34 -0800 Subject: [PATCH 26/29] indexer.New takes ds.Key --- dsindex/indexer.go | 12 ++++++------ dsindex/indexer_test.go | 4 ++-- dspinner/pin.go | 38 ++++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/dsindex/indexer.go b/dsindex/indexer.go index 35a5e0f..e48af2e 100644 --- a/dsindex/indexer.go +++ b/dsindex/indexer.go @@ -32,10 +32,10 @@ type Indexer interface { // the values that were deleted. DeleteAll(ctx context.Context) (count int, err error) - // ForEach calls the function for each value in the specified index, until + // ForEach calls the function for each value in the specified key, until // there are no more values, or until the function returns false. If key // is empty string, then all keys are iterated. - ForEach(ctx context.Context, index string, fn func(key, value string) bool) error + ForEach(ctx context.Context, key string, fn func(key, value string) bool) error // HasValue determines if the key contains the specified value HasValue(ctx context.Context, key, value string) (bool, error) @@ -61,9 +61,9 @@ type indexer struct { // // To persist the actions of calling Indexer functions, it is necessary to call // dstore.Sync. -func New(dstore ds.Datastore, name string) Indexer { +func New(dstore ds.Datastore, name ds.Key) Indexer { return &indexer{ - dstore: namespace.Wrap(dstore, ds.NewKey(name)), + dstore: namespace.Wrap(dstore, name), } } @@ -188,8 +188,8 @@ func (x *indexer) Search(ctx context.Context, key string) ([]string, error) { } // SyncIndex synchronizes the keys in the target Indexer to match those of the -// ref Indexer. The name portion of the stored data is not synchronized, only -// the key/value portion of the indexes. +// ref Indexer. This function does not change this indexer's key root (name +// passed into New). func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) { // Build reference index map refs := map[string]string{} diff --git a/dsindex/indexer_test.go b/dsindex/indexer_test.go index dcea2c1..45372c6 100644 --- a/dsindex/indexer_test.go +++ b/dsindex/indexer_test.go @@ -9,7 +9,7 @@ import ( func createIndexer() Indexer { dstore := ds.NewMapDatastore() - nameIndex := New(dstore, "/data/nameindex") + nameIndex := New(dstore, ds.NewKey("/data/nameindex")) ctx := context.Background() nameIndex.Add(ctx, "alice", "a1") @@ -243,7 +243,7 @@ func TestSyncIndex(t *testing.T) { nameIndex := createIndexer() dstore := ds.NewMapDatastore() - refIndex := New(dstore, "/ref") + refIndex := New(dstore, ds.NewKey("/ref")) refIndex.Add(ctx, "alice", "a1") refIndex.Add(ctx, "cathy", "zz") refIndex.Add(ctx, "dennis", "d1") diff --git a/dspinner/pin.go b/dspinner/pin.go index d96f9e2..fbd5542 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -6,6 +6,7 @@ package dspinner import ( "bytes" "context" + "errors" "fmt" "path" "sync" @@ -31,7 +32,7 @@ const ( var ( // ErrNotPinned is returned when trying to unpin items that are not pinned. - ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly") + ErrNotPinned = errors.New("not pinned or pinned indirectly") log logging.StandardLogger = logging.Logger("pin") @@ -109,9 +110,9 @@ type syncDAGService interface { // there is no data present in the datastore, then an empty pinner is returned. func New(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (ipfspinner.Pinner, error) { p := &pinner{ - cidDIndex: dsindex.New(dstore, pinCidDIndexPath), - cidRIndex: dsindex.New(dstore, pinCidRIndexPath), - nameIndex: dsindex.New(dstore, pinNameIndexPath), + cidDIndex: dsindex.New(dstore, ds.NewKey(pinCidDIndexPath)), + cidRIndex: dsindex.New(dstore, ds.NewKey(pinCidRIndexPath)), + nameIndex: dsindex.New(dstore, ds.NewKey(pinNameIndexPath)), dserv: dserv, dstore: dstore, } @@ -656,9 +657,9 @@ func (p *pinner) loadAllPins(ctx context.Context) ([]*pin, error) { func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { // Build temporary in-memory CID index from pins dstoreMem := ds.NewMapDatastore() - tmpCidDIndex := dsindex.New(dstoreMem, pinCidDIndexPath) - tmpCidRIndex := dsindex.New(dstoreMem, pinCidRIndexPath) - tmpNameIndex := dsindex.New(dstoreMem, pinNameIndexPath) + tmpCidDIndex := dsindex.New(dstoreMem, ds.NewKey(pinCidDIndexPath)) + tmpCidRIndex := dsindex.New(dstoreMem, ds.NewKey(pinCidRIndexPath)) + tmpNameIndex := dsindex.New(dstoreMem, ds.NewKey(pinNameIndexPath)) var hasNames bool for _, pp := range pins { if ctx.Err() != nil { @@ -778,7 +779,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return err } if !found { - return fmt.Errorf("'from' cid was not recursively pinned already") + return errors.New("'from' cid was not recursively pinned already") } // If `from` already recursively pinned and `to` is the same, then all done @@ -792,7 +793,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return err } if found { - return fmt.Errorf("'to' cid was already recursively pinned") + return errors.New("'to' cid was already recursively pinned") } // Temporarily unlock while we fetch the differences. @@ -899,8 +900,9 @@ func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.C func encodePin(p *pin) ([]byte, error) { var buf bytes.Buffer encoder := cbor.NewMarshaller(&buf) + mode := int(p.mode) pinData := map[string]interface{}{ - "mode": p.mode, + "mode": mode, "cid": p.cid.Bytes(), } // Encode optional fields @@ -930,11 +932,11 @@ func decodePin(pid string, data []byte) (*pin, error) { cidData, ok := pinData["cid"] if !ok { - return nil, fmt.Errorf("missing cid") + return nil, errors.New("missing cid") } cidBytes, ok := cidData.([]byte) if !ok { - return nil, fmt.Errorf("invalid pin cid data") + return nil, errors.New("invalid pin cid data") } c, err := cid.Cast(cidBytes) if err != nil { @@ -943,16 +945,16 @@ func decodePin(pid string, data []byte) (*pin, error) { modeData, ok := pinData["mode"] if !ok { - return nil, fmt.Errorf("missing mode") + return nil, errors.New("missing mode") } - mode64, ok := modeData.(int) + mode, ok := modeData.(int) if !ok { - return nil, fmt.Errorf("invalid pin mode data") + return nil, errors.New("invalid pin mode data") } p := &pin{ id: pid, - mode: ipfspinner.Mode(mode64), + mode: ipfspinner.Mode(int(mode)), cid: c, } @@ -962,7 +964,7 @@ func decodePin(pid string, data []byte) (*pin, error) { if ok && meta != nil { p.metadata, ok = meta.(map[string]interface{}) if !ok { - return nil, fmt.Errorf("cannot decode metadata") + return nil, errors.New("cannot decode metadata") } } @@ -970,7 +972,7 @@ func decodePin(pid string, data []byte) (*pin, error) { if ok && name != nil { p.name, ok = name.(string) if !ok { - return nil, fmt.Errorf("invalid pin name data") + return nil, errors.New("invalid pin name data") } } From 9c335cd6830ffb01e9b8a877def7efc5ad56cd13 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 24 Nov 2020 13:21:04 -0800 Subject: [PATCH 27/29] Change pin encoding. Add unit test --- dspinner/pin.go | 33 +++++++++++++++-------------- dspinner/pin_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index fbd5542..ce14c69 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -194,7 +194,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { p.removePinsForCid(ctx, c, ipfspinner.Direct) } - err = p.addPin(ctx, c, ipfspinner.Recursive, "") + _, err = p.addPin(ctx, c, ipfspinner.Recursive, "") if err != nil { return err } @@ -207,7 +207,7 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return fmt.Errorf("%s already pinned recursively", c.String()) } - err = p.addPin(ctx, c, ipfspinner.Direct, "") + _, err = p.addPin(ctx, c, ipfspinner.Direct, "") if err != nil { return err } @@ -215,14 +215,14 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { return nil } -func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) error { +func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) (string, error) { // Create new pin and store in datastore pp := newPin(c, mode, name) // Serialize pin pinData, err := encodePin(pp) if err != nil { - return fmt.Errorf("could not encode pin: %v", err) + return "", fmt.Errorf("could not encode pin: %v", err) } p.setDirty(ctx, true) @@ -237,14 +237,14 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na panic("pin mode must be recursive or direct") } if err != nil { - return fmt.Errorf("could not add pin cid index: %v", err) + return "", fmt.Errorf("could not add pin cid index: %v", err) } if name != "" { // Store name index err = p.nameIndex.Add(ctx, name, pp.id) if err != nil { - return fmt.Errorf("could not add pin name index: %v", err) + return "", fmt.Errorf("could not add pin name index: %v", err) } } @@ -259,10 +259,10 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na if name != "" { p.nameIndex.Delete(ctx, name, pp.id) } - return err + return "", err } - return nil + return pp.id, nil } func (p *pinner) removePin(ctx context.Context, pp *pin) error { @@ -805,7 +805,7 @@ func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error return err } - err = p.addPin(ctx, to, ipfspinner.Recursive, "") + _, err = p.addPin(ctx, to, ipfspinner.Recursive, "") if err != nil { return err } @@ -865,7 +865,7 @@ func (p *pinner) PinWithMode(c cid.Cid, mode ipfspinner.Mode) { panic("unrecognized pin mode") } - err := p.addPin(ctx, c, mode, "") + _, err := p.addPin(ctx, c, mode, "") if err != nil { return } @@ -898,11 +898,9 @@ func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.C } func encodePin(p *pin) ([]byte, error) { - var buf bytes.Buffer - encoder := cbor.NewMarshaller(&buf) - mode := int(p.mode) + modeB := []byte{byte(p.mode)} pinData := map[string]interface{}{ - "mode": mode, + "mode": modeB, "cid": p.cid.Bytes(), } // Encode optional fields @@ -913,6 +911,8 @@ func encodePin(p *pin) ([]byte, error) { pinData["metadata"] = p.metadata } + var buf bytes.Buffer + encoder := cbor.NewMarshaller(&buf) err := encoder.Marshal(pinData) if err != nil { return nil, err @@ -947,14 +947,13 @@ func decodePin(pid string, data []byte) (*pin, error) { if !ok { return nil, errors.New("missing mode") } - mode, ok := modeData.(int) + modeB, ok := modeData.([]byte) if !ok { return nil, errors.New("invalid pin mode data") } - p := &pin{ id: pid, - mode: ipfspinner.Mode(int(mode)), + mode: ipfspinner.Mode(int(modeB[0])), cid: c, } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index c7bd766..cbb4419 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -294,6 +294,12 @@ func TestPinnerBasic(t *testing.T) { if err != nil { t.Fatal(err) } + if pp.mode != ipfspin.Direct { + t.Error("loaded pin has wrong mode") + } + if pp.cid != ak { + t.Error("loaded pin has wrong cid") + } err = dsp.dstore.Delete(pp.dsKey()) if err != nil { t.Fatal(err) @@ -316,6 +322,49 @@ func TestPinnerBasic(t *testing.T) { log = realLog } +func TestAddLoadPin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + bstore := blockstore.NewBlockstore(dstore) + bserv := bs.New(bstore, offline.Exchange(bstore)) + + dserv := mdag.NewDAGService(bserv) + + ipfsPin, err := New(ctx, dstore, dserv) + if err != nil { + t.Fatal(err) + } + + p := ipfsPin.(*pinner) + + a, ak := randNode() + dserv.Add(ctx, a) + + mode := ipfspin.Recursive + name := "my-pin" + pid, err := p.addPin(ctx, ak, mode, name) + if err != nil { + t.Fatal(err) + } + + // Load pin and check that data decoded correctly + pinData, err := p.loadPin(ctx, pid) + if err != nil { + t.Fatal(err) + } + if pinData.mode != mode { + t.Error("worng pin mode") + } + if pinData.cid != ak { + t.Error("wrong pin cid") + } + if pinData.name != name { + t.Error("wrong pin name; expected", name, "got", pinData.name) + } +} + func TestRemovePinWithMode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 9fafc5180f4fd0730580ea66a0fb15f12603360e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 30 Nov 2020 06:14:49 -0500 Subject: [PATCH 28/29] switch to atlas pin encoding --- dspinner/pin.go | 156 ++++++++++++++++--------------------------- dspinner/pin_test.go | 42 ++++++------ 2 files changed, 78 insertions(+), 120 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index ce14c69..13e8e21 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -4,7 +4,6 @@ package dspinner import ( - "bytes" "context" "errors" "fmt" @@ -21,6 +20,7 @@ import ( mdag "github.com/ipfs/go-merkledag" "github.com/ipfs/go-merkledag/dagutils" "github.com/polydawn/refmt/cbor" + "github.com/polydawn/refmt/obj/atlas" ) const ( @@ -43,6 +43,8 @@ var ( pinNameIndexPath string dirtyKey = ds.NewKey(dirtyKeyPath) + + pinAtl atlas.Atlas ) func init() { @@ -61,6 +63,26 @@ func init() { pinCidRIndexPath = path.Join(indexKeyPath, "cidRindex") pinCidDIndexPath = path.Join(indexKeyPath, "cidDindex") pinNameIndexPath = path.Join(indexKeyPath, "nameIndex") + + pinAtl = atlas.MustBuild( + atlas.BuildEntry(pin{}).StructMap(). + AddField("Cid", atlas.StructMapEntry{SerialName: "cid"}). + AddField("Metadata", atlas.StructMapEntry{SerialName: "metadata", OmitEmpty: true}). + AddField("Mode", atlas.StructMapEntry{SerialName: "mode"}). + AddField("Name", atlas.StructMapEntry{SerialName: "name", OmitEmpty: true}). + Complete(), + atlas.BuildEntry(cid.Cid{}).Transform(). + TransformMarshal(atlas.MakeMarshalTransformFunc(func(live cid.Cid) ([]byte, error) { return live.MarshalBinary() })). + TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(func(serializable []byte) (cid.Cid, error) { + c := cid.Cid{} + err := c.UnmarshalBinary(serializable) + if err != nil { + return cid.Cid{}, err + } + return c, nil + })).Complete(), + ) + pinAtl = pinAtl.WithMapMorphism(atlas.MapMorphism{KeySortMode: atlas.KeySortMode_Strings}) } // pinner implements the Pinner interface @@ -81,23 +103,23 @@ type pinner struct { var _ ipfspinner.Pinner = (*pinner)(nil) type pin struct { - id string - cid cid.Cid - metadata map[string]interface{} - mode ipfspinner.Mode - name string + Id string `refmt:"id,ignore` + Cid cid.Cid `refmt:"cid"` + Metadata map[string]interface{} `refmt:"metadata,omitempty"` + Mode ipfspinner.Mode `refmt:"mode"` + Name string `refmt:"name,omitempty"` } func (p *pin) dsKey() ds.Key { - return ds.NewKey(path.Join(pinKeyPath, p.id)) + return ds.NewKey(path.Join(pinKeyPath, p.Id)) } func newPin(c cid.Cid, mode ipfspinner.Mode, name string) *pin { return &pin{ - id: ds.RandomKey().String(), - cid: c, - name: name, - mode: mode, + Id: ds.RandomKey().String(), + Cid: c, + Name: name, + Mode: mode, } } @@ -230,9 +252,9 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na // Store CID index switch mode { case ipfspinner.Recursive: - err = p.cidRIndex.Add(ctx, c.KeyString(), pp.id) + err = p.cidRIndex.Add(ctx, c.KeyString(), pp.Id) case ipfspinner.Direct: - err = p.cidDIndex.Add(ctx, c.KeyString(), pp.id) + err = p.cidDIndex.Add(ctx, c.KeyString(), pp.Id) default: panic("pin mode must be recursive or direct") } @@ -242,7 +264,7 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na if name != "" { // Store name index - err = p.nameIndex.Add(ctx, name, pp.id) + err = p.nameIndex.Add(ctx, name, pp.Id) if err != nil { return "", fmt.Errorf("could not add pin name index: %v", err) } @@ -252,17 +274,17 @@ func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, na err = p.dstore.Put(pp.dsKey(), pinData) if err != nil { if mode == ipfspinner.Recursive { - p.cidRIndex.Delete(ctx, c.KeyString(), pp.id) + p.cidRIndex.Delete(ctx, c.KeyString(), pp.Id) } else { - p.cidDIndex.Delete(ctx, c.KeyString(), pp.id) + p.cidDIndex.Delete(ctx, c.KeyString(), pp.Id) } if name != "" { - p.nameIndex.Delete(ctx, name, pp.id) + p.nameIndex.Delete(ctx, name, pp.Id) } return "", err } - return pp.id, nil + return pp.Id, nil } func (p *pinner) removePin(ctx context.Context, pp *pin) error { @@ -275,18 +297,18 @@ func (p *pinner) removePin(ctx context.Context, pp *pin) error { return err } // Remove cid index from datastore - if pp.mode == ipfspinner.Recursive { - err = p.cidRIndex.Delete(ctx, pp.cid.KeyString(), pp.id) + if pp.Mode == ipfspinner.Recursive { + err = p.cidRIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) } else { - err = p.cidDIndex.Delete(ctx, pp.cid.KeyString(), pp.id) + err = p.cidDIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id) } if err != nil { return err } - if pp.name != "" { + if pp.Name != "" { // Remove name index from datastore - err = p.nameIndex.Delete(ctx, pp.name, pp.id) + err = p.nameIndex.Delete(ctx, pp.Name, pp.Id) if err != nil { return err } @@ -599,7 +621,7 @@ func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinne } return false, err } - if mode == ipfspinner.Any || pp.mode == mode { + if mode == ipfspinner.Any || pp.Mode == mode { err = p.removePin(ctx, pp) if err != nil { return false, err @@ -665,13 +687,13 @@ func (p *pinner) rebuildIndexes(ctx context.Context, pins []*pin) error { if ctx.Err() != nil { return ctx.Err() } - if pp.mode == ipfspinner.Recursive { - tmpCidRIndex.Add(ctx, pp.cid.KeyString(), pp.id) - } else if pp.mode == ipfspinner.Direct { - tmpCidDIndex.Add(ctx, pp.cid.KeyString(), pp.id) + if pp.Mode == ipfspinner.Recursive { + tmpCidRIndex.Add(ctx, pp.Cid.KeyString(), pp.Id) + } else if pp.Mode == ipfspinner.Direct { + tmpCidDIndex.Add(ctx, pp.Cid.KeyString(), pp.Id) } - if pp.name != "" { - tmpNameIndex.Add(ctx, pp.name, pp.id) + if pp.Name != "" { + tmpNameIndex.Add(ctx, pp.Name, pp.Id) hasNames = true } } @@ -898,83 +920,19 @@ func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.C } func encodePin(p *pin) ([]byte, error) { - modeB := []byte{byte(p.mode)} - pinData := map[string]interface{}{ - "mode": modeB, - "cid": p.cid.Bytes(), - } - // Encode optional fields - if p.name != "" { - pinData["name"] = p.name - } - if len(p.metadata) != 0 { - pinData["metadata"] = p.metadata - } - - var buf bytes.Buffer - encoder := cbor.NewMarshaller(&buf) - err := encoder.Marshal(pinData) + b, err := cbor.MarshalAtlased(p, pinAtl) if err != nil { return nil, err } - return buf.Bytes(), nil + return b, nil } func decodePin(pid string, data []byte) (*pin, error) { - reader := bytes.NewReader(data) - decoder := cbor.NewUnmarshaller(cbor.DecodeOptions{}, reader) - - var pinData map[string]interface{} - err := decoder.Unmarshal(&pinData) - if err != nil { - return nil, fmt.Errorf("cannot decode pin: %v", err) - } - - cidData, ok := pinData["cid"] - if !ok { - return nil, errors.New("missing cid") - } - cidBytes, ok := cidData.([]byte) - if !ok { - return nil, errors.New("invalid pin cid data") - } - c, err := cid.Cast(cidBytes) + p := &pin{Id: pid} + err := cbor.UnmarshalAtlased(cbor.DecodeOptions{}, data, p, pinAtl) if err != nil { - return nil, fmt.Errorf("cannot decode pin cid: %v", err) - } - - modeData, ok := pinData["mode"] - if !ok { - return nil, errors.New("missing mode") - } - modeB, ok := modeData.([]byte) - if !ok { - return nil, errors.New("invalid pin mode data") - } - p := &pin{ - id: pid, - mode: ipfspinner.Mode(int(modeB[0])), - cid: c, - } - - // Decode optional data - - meta, ok := pinData["metadata"] - if ok && meta != nil { - p.metadata, ok = meta.(map[string]interface{}) - if !ok { - return nil, errors.New("cannot decode metadata") - } - } - - name, ok := pinData["name"] - if ok && name != nil { - p.name, ok = name.(string) - if !ok { - return nil, errors.New("invalid pin name data") - } + return nil, err } - return p, nil } diff --git a/dspinner/pin_test.go b/dspinner/pin_test.go index cbb4419..40e2c70 100644 --- a/dspinner/pin_test.go +++ b/dspinner/pin_test.go @@ -294,10 +294,10 @@ func TestPinnerBasic(t *testing.T) { if err != nil { t.Fatal(err) } - if pp.mode != ipfspin.Direct { + if pp.Mode != ipfspin.Direct { t.Error("loaded pin has wrong mode") } - if pp.cid != ak { + if pp.Cid != ak { t.Error("loaded pin has wrong cid") } err = dsp.dstore.Delete(pp.dsKey()) @@ -354,14 +354,14 @@ func TestAddLoadPin(t *testing.T) { if err != nil { t.Fatal(err) } - if pinData.mode != mode { + if pinData.Mode != mode { t.Error("worng pin mode") } - if pinData.cid != ak { + if pinData.Cid != ak { t.Error("wrong pin cid") } - if pinData.name != name { - t.Error("wrong pin name; expected", name, "got", pinData.name) + if pinData.Name != name { + t.Error("wrong pin name; expected", name, "got", pinData.Name) } } @@ -720,36 +720,36 @@ func TestEncodeDecodePin(t *testing.T) { _, c := randNode() pin := newPin(c, ipfspin.Recursive, "testpin") - pin.metadata = make(map[string]interface{}, 2) - pin.metadata["hello"] = "world" - pin.metadata["foo"] = "bar" + pin.Metadata = make(map[string]interface{}, 2) + pin.Metadata["hello"] = "world" + pin.Metadata["foo"] = "bar" encBytes, err := encodePin(pin) if err != nil { t.Fatal(err) } - decPin, err := decodePin(pin.id, encBytes) + decPin, err := decodePin(pin.Id, encBytes) if err != nil { t.Fatal(err) } - if decPin.id != pin.id { - t.Errorf("wrong pin id: expect %q got %q", pin.id, decPin.id) + if decPin.Id != pin.Id { + t.Errorf("wrong pin id: expect %q got %q", pin.Id, decPin.Id) } - if decPin.cid != pin.cid { - t.Errorf("wrong pin cid: expect %q got %q", pin.cid.String(), decPin.cid.String()) + if decPin.Cid != pin.Cid { + t.Errorf("wrong pin cid: expect %q got %q", pin.Cid.String(), decPin.Cid.String()) } - if decPin.mode != pin.mode { - expect, _ := ipfspin.ModeToString(pin.mode) - got, _ := ipfspin.ModeToString(decPin.mode) + if decPin.Mode != pin.Mode { + expect, _ := ipfspin.ModeToString(pin.Mode) + got, _ := ipfspin.ModeToString(decPin.Mode) t.Errorf("wrong pin mode: expect %s got %s", expect, got) } - if decPin.name != pin.name { - t.Errorf("wrong pin name: expect %q got %q", pin.name, decPin.name) + if decPin.Name != pin.Name { + t.Errorf("wrong pin name: expect %q got %q", pin.Name, decPin.Name) } - for key, val := range pin.metadata { - dval, ok := decPin.metadata[key] + for key, val := range pin.Metadata { + dval, ok := decPin.Metadata[key] if !ok { t.Errorf("decoded pin missing metadata key %q", key) } From 2586c60bcee64bdaf63d9a56c9528f0c90d155e2 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 30 Nov 2020 13:19:16 -0500 Subject: [PATCH 29/29] removed type annotations from pin struct --- dspinner/pin.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dspinner/pin.go b/dspinner/pin.go index 13e8e21..5fd65e7 100644 --- a/dspinner/pin.go +++ b/dspinner/pin.go @@ -103,11 +103,11 @@ type pinner struct { var _ ipfspinner.Pinner = (*pinner)(nil) type pin struct { - Id string `refmt:"id,ignore` - Cid cid.Cid `refmt:"cid"` - Metadata map[string]interface{} `refmt:"metadata,omitempty"` - Mode ipfspinner.Mode `refmt:"mode"` - Name string `refmt:"name,omitempty"` + Id string + Cid cid.Cid + Metadata map[string]interface{} + Mode ipfspinner.Mode + Name string } func (p *pin) dsKey() ds.Key {