fix: use @ipld/dag-pb instead of ipld-dag-pb (#116)

Swaps out the old ipld stack for the new multiformats stack.  Removes dependency on the js-ipfs API to enable easier reuse.

BREAKING CHANGE: uses new multiformats stack and takes a blockservice instead of the block api

Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: achingbrain <alex@achingbrain.net>

.travis.yml

@@ -1,5 +1,6 @@
 language: node_js
 cache: npm
+dist: focal
 
 branches:
   only:

packages/ipfs-unixfs-exporter/.aegir.js

@@ -23,6 +23,7 @@ const buildConfig = {
 /** @type {import('aegir').PartialOptions} */
 module.exports = {
   build: {
+    bundlesizeMax: '34KB',
     config: buildConfig
   },
   test: {

packages/ipfs-unixfs-exporter/package.json

@@ -12,9 +12,9 @@
     "test": "aegir test",
     "build": "aegir build",
     "clean": "rimraf ./dist",
-    "lint": "aegir ts --check && aegir lint",
+    "lint": "aegir ts -p check && aegir lint",
     "coverage": "nyc -s npm run test -t node && nyc report --reporter=html",
-    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types -i copy -i util -i crypto-browserify -i events -i readable-stream"
+    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i copy -i util -i crypto-browserify -i events -i readable-stream -i interface-blockstore"
   },
   "repository": {
     "type": "git",
@@ -36,37 +36,36 @@
     "@types/mocha": "^8.2.1",
     "@types/sinon": "^10.0.0",
     "abort-controller": "^3.0.0",
-    "aegir": "^33.1.0",
+    "aegir": "^34.0.0",
     "copy": "^0.3.2",
     "crypto-browserify": "^3.12.0",
     "detect-node": "^2.0.4",
     "events": "^3.3.0",
-    "ipfs-core-types": "^0.3.1",
     "ipfs-unixfs-importer": "^7.0.3",
-    "ipld": "^0.29.0",
-    "ipld-block": "^0.11.1",
-    "ipld-dag-pb": "^0.22.2",
-    "ipld-in-memory": "^8.0.0",
     "it-all": "^1.0.5",
     "it-buffer-stream": "^2.0.0",
     "it-first": "^1.0.6",
     "merge-options": "^3.0.4",
-    "multicodec": "^3.0.1",
+    "murmurhash3js-revisited": "^3.0.0",
     "native-abort-controller": "^1.0.3",
     "nyc": "^15.0.0",
     "readable-stream": "^3.6.0",
     "rimraf": "^3.0.2",
-    "sinon": "^10.0.0",
+    "sinon": "^11.1.1",
     "uint8arrays": "^2.1.2",
     "util": "^0.12.3"
   },
   "dependencies": {
-    "cids": "^1.1.5",
+    "@ipld/dag-cbor": "^6.0.4",
+    "@ipld/dag-pb": "^2.0.2",
     "err-code": "^3.0.1",
     "hamt-sharding": "^2.0.0",
+    "interface-blockstore": "^1.0.0",
     "ipfs-unixfs": "^4.0.3",
     "it-last": "^1.0.5",
-    "multihashing-async": "^2.1.0"
+    "multiformats": "^9.0.4",
+    "murmurhash3js-revisited": "^3.0.0",
+    "uint8arrays": "^2.1.5"
   },
   "types": "dist/src/index.d.ts",
   "files": [

packages/ipfs-unixfs-exporter/src/index.js

@@ -1,14 +1,13 @@
 'use strict'
 
 const errCode = require('err-code')
-const CID = require('cids')
+const { CID } = require('multiformats/cid')
 const resolve = require('./resolvers')
 const last = require('it-last')
 
 /**
  * @typedef {import('ipfs-unixfs').UnixFS} UnixFS
- * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
- * @typedef {import('ipld')} IPLD
+ * @typedef {import('interface-blockstore').Blockstore} Blockstore
  * @typedef {import('./types').ExporterOptions} ExporterOptions
  * @typedef {import('./types').UnixFSFile} UnixFSFile
  * @typedef {import('./types').UnixFSDirectory} UnixFSDirectory
@@ -32,14 +31,15 @@ const toPathComponents = (path = '') => {
 const cidAndRest = (path) => {
   if (path instanceof Uint8Array) {
     return {
-      cid: new CID(path),
+      cid: CID.decode(path),
       toResolve: []
     }
   }
 
-  if (CID.isCID(path)) {
+  const cid = CID.asCID(path)
+  if (cid) {
     return {
-      cid: path,
+      cid,
       toResolve: []
     }
   }
@@ -52,7 +52,7 @@ const cidAndRest = (path) => {
     const output = toPathComponents(path)
 
     return {
-      cid: new CID(output[0]),
+      cid: CID.parse(output[0]),
       toResolve: output.slice(1)
     }
   }
@@ -62,10 +62,10 @@ const cidAndRest = (path) => {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function * walkPath (path, ipld, options = {}) {
+async function * walkPath (path, blockstore, options = {}) {
   let {
     cid,
     toResolve
@@ -75,7 +75,7 @@ async function * walkPath (path, ipld, options = {}) {
   const startingDepth = toResolve.length
 
   while (true) {
-    const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld, options)
+    const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockstore, options)
 
     if (!result.entry && !result.next) {
       throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -99,11 +99,11 @@ async function * walkPath (path, ipld, options = {}) {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function exporter (path, ipld, options = {}) {
-  const result = await last(walkPath(path, ipld, options))
+async function exporter (path, blockstore, options = {}) {
+  const result = await last(walkPath(path, blockstore, options))
 
   if (!result) {
     throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -114,11 +114,11 @@ async function exporter (path, ipld, options = {}) {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {Blockstore} blockstore
  * @param {ExporterOptions} [options]
  */
-async function * recursive (path, ipld, options = {}) {
-  const node = await exporter(path, ipld, options)
+async function * recursive (path, blockstore, options = {}) {
+  const node = await exporter(path, blockstore, options)
 
   if (!node) {
     return

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -1,7 +1,8 @@
 'use strict'
 
-const CID = require('cids')
+const { CID } = require('multiformats/cid')
 const errCode = require('err-code')
+const dagCbor = require('@ipld/dag-cbor')
 
 /**
  * @typedef {import('../types').Resolver} Resolver
@@ -10,9 +11,9 @@ const errCode = require('err-code')
 /**
  * @type {Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
-  const object = await ipld.get(cid, options)
-  const block = await ipld.get(new CID(1, 'raw', cid.multihash))
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
+  const block = await blockstore.get(cid)
+  const object = dagCbor.decode(block)
   let subObject = object
   let subPath = path
 
@@ -24,7 +25,8 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       toResolve.shift()
       subPath = `${subPath}/${prop}`
 
-      if (CID.isCID(subObject[prop])) {
+      const subObjectCid = CID.asCID(subObject[prop])
+      if (subObjectCid) {
         return {
           entry: {
             type: 'object',
@@ -39,7 +41,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
             }
           },
           next: {
-            cid: subObject[prop],
+            cid: subObjectCid,
             name: prop,
             path: subPath,
             toResolve

packages/ipfs-unixfs-exporter/src/resolvers/identity.js

@@ -3,7 +3,7 @@
 const errCode = require('err-code')
 const extractDataFromBlock = require('../utils/extract-data-from-block')
 const validateOffsetAndLength = require('../utils/validate-offset-and-length')
-const mh = require('multihashing-async').multihash
+const mh = require('multiformats/hashes/digest')
 
 /**
  * @typedef {import('../types').ExporterOptions} ExporterOptions
@@ -32,12 +32,11 @@ const rawContent = (node) => {
 /**
  * @type {Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
   if (toResolve.length) {
     throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
   }
-
-  const buf = await mh.decode(cid.multihash)
+  const buf = await mh.decode(cid.multihash.bytes)
 
   return {
     entry: {
@@ -47,7 +46,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       cid,
       content: rawContent(buf.digest),
       depth,
-      size: buf.length,
+      size: buf.digest.length,
       node: buf.digest
     }
   }

packages/ipfs-unixfs-exporter/src/resolvers/index.js

@@ -2,11 +2,12 @@
 
 const errCode = require('err-code')
 
+const dagPb = require('@ipld/dag-pb')
+const dagCbor = require('@ipld/dag-cbor')
+const raw = require('multiformats/codecs/raw')
+const { identity } = require('multiformats/hashes/identity')
+
 /**
- * @typedef {import('cids')} CID
- * @typedef {import('ipld')} IPLD
- * @typedef {import('../types').ExporterOptions} ExporterOptions
- * @typedef {import('../types').UnixFSEntry} UnixFSEntry
  * @typedef {import('../types').Resolver} Resolver
  * @typedef {import('../types').Resolve} Resolve
  */
@@ -15,23 +16,23 @@ const errCode = require('err-code')
  * @type {{ [ key: string ]: Resolver }}
  */
 const resolvers = {
-  'dag-pb': require('./unixfs-v1'),
-  raw: require('./raw'),
-  'dag-cbor': require('./dag-cbor'),
-  identity: require('./identity')
+  [dagPb.code]: require('./unixfs-v1'),
+  [raw.code]: require('./raw'),
+  [dagCbor.code]: require('./dag-cbor'),
+  [identity.code]: require('./identity')
 }
 
 /**
  * @type {Resolve}
  */
-function resolve (cid, name, path, toResolve, depth, ipld, options) {
-  const resolver = resolvers[cid.codec]
+function resolve (cid, name, path, toResolve, depth, blockstore, options) {
+  const resolver = resolvers[cid.code]
 
   if (!resolver) {
-    throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ERR_NO_RESOLVER')
+    throw errCode(new Error(`No resolver for code ${cid.code}`), 'ERR_NO_RESOLVER')
   }
 
-  return resolver(cid, name, path, toResolve, resolve, depth, ipld, options)
+  return resolver(cid, name, path, toResolve, resolve, depth, blockstore, options)
 }
 
 module.exports = resolve

packages/ipfs-unixfs-exporter/src/resolvers/raw.js

@@ -30,23 +30,23 @@ const rawContent = (node) => {
 /**
  * @type {import('../types').Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => {
   if (toResolve.length) {
     throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND')
   }
 
-  const buf = await ipld.get(cid, options)
+  const block = await blockstore.get(cid, options)
 
   return {
     entry: {
       type: 'raw',
       name,
       path,
       cid,
-      content: rawContent(buf),
+      content: rawContent(block),
       depth,
-      size: buf.length,
-      node: buf
+      size: block.length,
+      node: block
     }
   }
 }

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js

@@ -9,7 +9,7 @@
 /**
  * @type {UnixfsV1Resolver}
  */
-const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
+const directoryContent = (cid, node, unixfs, path, resolve, depth, blockstore) => {
   /**
    * @param {ExporterOptions} [options]
    * @returns {UnixfsV1DirectoryContent}
@@ -20,7 +20,7 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
     const links = node.Links.slice(offset, length)
 
     for (const link of links) {
-      const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, ipld, options)
+      const result = await resolve(link.Hash, link.Name || '', `${path}/${link.Name || ''}`, [], depth + 1, blockstore, options)
 
       if (result.entry) {
         yield result.entry