Merge pull request #5285 from ipfs/kevina/cidv0v1

Add ability to retrieve blocks even if given using a different CID version

core/builder.go

@@ -13,6 +13,7 @@ import (
 	pin "github.com/ipfs/go-ipfs/pin"
 	repo "github.com/ipfs/go-ipfs/repo"
 	cfg "github.com/ipfs/go-ipfs/repo/config"
+	cidv0v1 "github.com/ipfs/go-ipfs/thirdparty/cidv0v1"
 	"github.com/ipfs/go-ipfs/thirdparty/verifbs"
 	uio "gx/ipfs/QmWdTRLi3H7ZJQ8s7NYo8oitz5JHEEPKLn1QPMsJVWg2Ew/go-unixfs/io"
 	dag "gx/ipfs/Qma2BR57Wqp8w9vPreK4dEzoXXk8DFFRL3LresMZg4QpzN/go-merkledag"
@@ -211,6 +212,8 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 
 	wbs = bstore.NewIdStore(wbs)
 
+	wbs = cidv0v1.NewBlockstore(wbs)
+
 	n.BaseBlocks = wbs
 	n.GCLocker = bstore.NewGCLocker()
 	n.Blockstore = bstore.NewGCBlockstore(wbs, n.GCLocker)

test/sharness/t0276-cidv0v1.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2017 Jakub Sztandera
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="CID Version 0/1 Duality"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+#
+#
+#
+
+test_expect_success "create two small files" '
+  random 1000 7 > afile
+  random 1000 9 > bfile
+'
+
+test_expect_success "add file using CIDv1 but don't pin" '
+  AHASHv1=$(ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile)
+'
+
+test_expect_success "add file using CIDv0" '
+  AHASHv0=$(ipfs add -q --cid-version=0 afile)
+'
+
+test_expect_success "check hashes" '
+  test "$(cid-fmt %v-%c $AHASHv0)" = "cidv0-protobuf" &&
+  test "$(cid-fmt %v-%c $AHASHv1)" = "cidv1-protobuf" &&
+  test "$(cid-fmt -v 0 %s $AHASHv1)" = "$AHASHv0"
+'
+
+test_expect_success "make sure CIDv1 hash really is in the repo" '
+  ipfs refs local | grep -q $AHASHv1
+'
+
+test_expect_success "make sure CIDv0 hash really is in the repo" '
+  ipfs refs local | grep -q $AHASHv0
+'
+
+test_expect_success "run gc" '
+  ipfs repo gc
+'
+
+test_expect_success "make sure the CIDv0 hash is in the repo" '
+  ipfs refs local | grep -q $AHASHv0
+'
+
+test_expect_success "make sure we can get CIDv0 added file" '
+  ipfs cat $AHASHv0 > thefile &&
+  test_cmp afile thefile
+'
+
+test_expect_success "make sure the CIDv1 hash is not in the repo" '
+  ! ipfs refs local | grep -q $AHASHv1
+'
+
+test_expect_success "clean up" '
+  ipfs pin rm $AHASHv0 &&
+  ipfs repo gc &&
+  ! ipfs refs local | grep -q $AHASHv0
+'
+
+#
+#
+#
+
+test_expect_success "add file using CIDv1 but don't pin" '
+  ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile
+'
+
+test_expect_success "check that we can access the file when converted to CIDv0" '
+  ipfs cat $AHASHv0 > thefile &&
+  test_cmp afile thefile
+'
+
+test_expect_success "clean up" '
+  ipfs repo gc
+'
+
+test_expect_success "add file using CIDv0 but don't pin" '
+  ipfs add -q --cid-version=0 --raw-leaves=false --pin=false afile
+'
+
+test_expect_success "check that we can access the file when converted to CIDv1" '
+  ipfs cat $AHASHv1 > thefile &&
+  test_cmp afile thefile
+'
+
+#
+#
+#
+
+test_expect_success "set up iptb testbed" '
+  iptb init -n 2 -p 0 -f --bootstrap=none
+'
+
+test_expect_success "start nodes" '
+  iptb start &&
+  iptb connect 0 1
+'
+
+test_expect_success "add afile using CIDv0 to node 0" '
+  iptb run 0 ipfs add -q --cid-version=0 afile
+'
+
+test_expect_success "get afile using CIDv1 via node 1" '
+  iptb run 1 ipfs --timeout=2s cat $AHASHv1 > thefile &&
+  test_cmp afile thefile
+'
+
+test_expect_success "add bfile using CIDv1 to node 0" '
+  BHASHv1=$(iptb run 0 ipfs add -q --cid-version=1 --raw-leaves=false bfile)
+'
+
+test_expect_success "get bfile using CIDv0 via node 1" '
+  BHASHv0=$(cid-fmt -v 0 %s $BHASHv1)
+  iptb run 1 ipfs --timeout=2s cat $BHASHv0 > thefile &&
+  test_cmp bfile thefile
+'
+
+test_expect_success "stop testbed" '
+  iptb stop
+'
+
+test_done

thirdparty/cidv0v1/blockstore.go

@@ -0,0 +1,72 @@
+package cidv0v1
+
+import (
+	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
+	blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
+	cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
+	bs "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
+)
+
+type blockstore struct {
+	bs.Blockstore
+}
+
+func NewBlockstore(b bs.Blockstore) bs.Blockstore {
+	return &blockstore{b}
+}
+
+func (b *blockstore) Has(c *cid.Cid) (bool, error) {
+	have, err := b.Blockstore.Has(c)
+	if have || err != nil {
+		return have, err
+	}
+	c1 := tryOtherCidVersion(c)
+	if c1 == nil {
+		return false, nil
+	}
+	return b.Blockstore.Has(c1)
+}
+
+func (b *blockstore) Get(c *cid.Cid) (blocks.Block, error) {
+	block, err := b.Blockstore.Get(c)
+	if err == nil {
+		return block, nil
+	}
+	if err != bs.ErrNotFound {
+		return nil, err
+	}
+	c1 := tryOtherCidVersion(c)
+	if c1 == nil {
+		return nil, bs.ErrNotFound
+	}
+	block, err = b.Blockstore.Get(c1)
+	if err != nil {
+		return nil, err
+	}
+	// modify block so it has the original CID
+	block, err = blocks.NewBlockWithCid(block.RawData(), c)
+	if err != nil {
+		return nil, err
+	}
+	// insert the block with the original CID to avoid problems
+	// with pinning
+	err = b.Blockstore.Put(block)
+	if err != nil {
+		return nil, err
+	}
+	return block, nil
+}
+
+func tryOtherCidVersion(c *cid.Cid) *cid.Cid {
+	prefix := c.Prefix()
+	if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 {
+		return nil
+	}
+	var c1 *cid.Cid
+	if prefix.Version == 0 {
+		c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash())
+	} else {
+		c1 = cid.NewCidV0(c.Hash())
+	}
+	return c1
+}