From 10695875fa69e3b123da997806cebd2b6228d810 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 18 Jan 2019 11:34:14 +0000 Subject: [PATCH 1/2] feat: wip load IPLD formats lazily from IPFS License: MIT Signed-off-by: Alan Shaw --- package.json | 1 + .../files-regular/cat-pull-stream.js | 3 +- src/core/index.js | 42 ++---------- src/core/runtime/ipld-formats-browser.js | 65 +++++++++++++++++++ src/core/runtime/ipld-formats-nodejs.js | 42 ++++++++++++ 5 files changed, 115 insertions(+), 38 deletions(-) create mode 100644 src/core/runtime/ipld-formats-browser.js create mode 100644 src/core/runtime/ipld-formats-nodejs.js diff --git a/package.json b/package.json index 0c0150f676..09b8f93ad6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js", "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js", "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js", + "./src/core/runtime/ipld-formats-nodejs.js": "./src/core/runtime/ipld-formats-browser.js", "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js", "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js", "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js", diff --git a/src/core/components/files-regular/cat-pull-stream.js b/src/core/components/files-regular/cat-pull-stream.js index 41c126e8d3..20b6d0fa3d 100644 --- a/src/core/components/files-regular/cat-pull-stream.js +++ b/src/core/components/files-regular/cat-pull-stream.js @@ -16,7 +16,8 @@ module.exports = function (self) { ipfsPath = normalizePath(ipfsPath) const pathComponents = ipfsPath.split('/') const restPath = normalizePath(pathComponents.slice(1).join('/')) - const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath) + const fileName = restPath.includes('/') ? restPath.split('/').pop() : restPath + const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath) || file.path === fileName if (options.preload !== false) { self._preload(pathComponents[0]) diff --git a/src/core/index.js b/src/core/index.js index 496396a443..ce08d5c138 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -23,40 +23,7 @@ const components = require('./components') const defaultRepo = require('./runtime/repo-nodejs') const preload = require('./preload') const mfsPreload = require('./mfs-preload') - -// All known (non-default) IPLD formats -const IpldFormats = { - get 'bitcoin-block' () { - return require('ipld-bitcoin') - }, - get 'eth-account-snapshot' () { - return require('ipld-ethereum').ethAccountSnapshot - }, - get 'eth-block' () { - return require('ipld-ethereum').ethBlock - }, - get 'eth-block-list' () { - return require('ipld-ethereum').ethBlockList - }, - get 'eth-state-trie' () { - return require('ipld-ethereum').ethStateTrie - }, - get 'eth-storage-trie' () { - return require('ipld-ethereum').ethStorageTrie - }, - get 'eth-tx' () { - return require('ipld-ethereum').ethTx - }, - get 'eth-tx-trie' () { - return require('ipld-ethereum').ethTxTrie - }, - get 'git-raw' () { - return require('ipld-git') - }, - get 'zcash-block' () { - return require('ipld-zcash') - } -} +const ipldFormats = require('./runtime/ipld-formats-nodejs') class IPFS extends EventEmitter { constructor (options) { @@ -109,6 +76,8 @@ class IPFS extends EventEmitter { CID: CID } + const loadIpldFormat = ipldFormats(this) + // IPFS Core Internals // this._repo - assigned above this._peerInfoBook = new PeerBook() @@ -119,9 +88,8 @@ class IPFS extends EventEmitter { this._ipld = new Ipld({ blockService: this._blockService, loadFormat: (codec, callback) => { - this.log('Loading IPLD format', codec) - if (IpldFormats[codec]) return callback(null, IpldFormats[codec]) - callback(new Error(`Missing IPLD format "${codec}"`)) + console.log('Loading IPLD format', codec) + loadIpldFormat(codec, callback) } }) this._preload = preload(this) diff --git a/src/core/runtime/ipld-formats-browser.js b/src/core/runtime/ipld-formats-browser.js new file mode 100644 index 0000000000..fc260101ad --- /dev/null +++ b/src/core/runtime/ipld-formats-browser.js @@ -0,0 +1,65 @@ +'use strict' + +const IPLD_FORMATS_CID = 'QmZaSX7mMriaBfvTvYdSS2kRdai3qUSMZ7edZnw4qzLPX2' + + + +module.exports = self => { + const IpldFormats = {} + + function createLoader (codec, path) { + + } + + // All known (non-default) IPLD formats + const IpldFormatLoaders = { + 'bitcoin-block' (callback) { + if (IpldFormats['bitcoin-block']) { + return callback(null, IpldFormats['bitcoin-block']) + } + + self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-bitcoin@0.1.9/index.min.js`, (err, data) => { + if (err) return callback(err) + const fn = new Function(data.toString()) // eslint-disable-line no-new-func + fn() + IpldFormats['bitcoin-block'] = window.IpldBitcoin + callback(null, IpldFormats['bitcoin-block']) + }) + }, + 'git-raw' (callback) { + if (IpldFormats['git-raw']) { + return callback(null, IpldFormats['git-raw']) + } + + console.log(`Loading from /ipfs/${IPLD_FORMATS_CID}/ipld-git@0.2.2/index.min.js`) + + self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-git@0.2.2/index.min.js`, (err, data) => { + if (err) return callback(err) + console.log('Loaded format git-raw') + const fn = new Function(data.toString()) // eslint-disable-line no-new-func + fn() + console.log('window.IpldGit', window.IpldGit) + IpldFormats['git-raw'] = window.IpldGit + callback(null, IpldFormats['git-raw']) + }) + }, + 'zcash-block' (callback) { + if (IpldFormats['zcash-block']) { + return callback(null, IpldFormats['zcash-block']) + } + + self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-zcash@0.1.6/index.min.js`, (err, data) => { + if (err) return callback(err) + const fn = new Function(data.toString()) // eslint-disable-line no-new-func + fn() + IpldFormats['zcash-block'] = window.IpldZcash + callback(null, IpldFormats['zcash-block']) + }) + } + } + + return (codec, callback) => { + if (IpldFormatLoaders[codec]) return IpldFormatLoaders[codec](callback) + callback(new Error(`Missing IPLD format "${codec}"`)) + } +} diff --git a/src/core/runtime/ipld-formats-nodejs.js b/src/core/runtime/ipld-formats-nodejs.js new file mode 100644 index 0000000000..e6560fe401 --- /dev/null +++ b/src/core/runtime/ipld-formats-nodejs.js @@ -0,0 +1,42 @@ +'use strict' + +// All known (non-default) IPLD formats +const IpldFormats = { + get 'bitcoin-block' () { + return require('ipld-bitcoin') + }, + get 'eth-account-snapshot' () { + return require('ipld-ethereum').ethAccountSnapshot + }, + get 'eth-block' () { + return require('ipld-ethereum').ethBlock + }, + get 'eth-block-list' () { + return require('ipld-ethereum').ethBlockList + }, + get 'eth-state-trie' () { + return require('ipld-ethereum').ethStateTrie + }, + get 'eth-storage-trie' () { + return require('ipld-ethereum').ethStorageTrie + }, + get 'eth-tx' () { + return require('ipld-ethereum').ethTx + }, + get 'eth-tx-trie' () { + return require('ipld-ethereum').ethTxTrie + }, + get 'git-raw' () { + return require('ipld-git') + }, + get 'zcash-block' () { + return require('ipld-zcash') + } +} + +module.exports = () => { + return (codec, callback) => { + if (IpldFormats[codec]) return callback(null, IpldFormats[codec]) + callback(new Error(`Missing IPLD format "${codec}"`)) + } +} From 974e60f9ff8c154781830e25de269c1219320836 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 18 Jan 2019 16:34:48 +0000 Subject: [PATCH 2/2] refactor: tidy up License: MIT Signed-off-by: Alan Shaw --- src/core/runtime/ipld-formats-browser.js | 130 ++++++++++++++--------- 1 file changed, 82 insertions(+), 48 deletions(-) diff --git a/src/core/runtime/ipld-formats-browser.js b/src/core/runtime/ipld-formats-browser.js index fc260101ad..077d0742ef 100644 --- a/src/core/runtime/ipld-formats-browser.js +++ b/src/core/runtime/ipld-formats-browser.js @@ -1,65 +1,99 @@ 'use strict' -const IPLD_FORMATS_CID = 'QmZaSX7mMriaBfvTvYdSS2kRdai3qUSMZ7edZnw4qzLPX2' - - +const IPLD_FORMATS_CID = 'QmP7qfTriY43hb2fUppkd5NvCSV5GxMAHM6cYHMq9uFTeX' module.exports = self => { - const IpldFormats = {} + const options = self._options.ipld || {} + const jsLoader = createIpfsJsLoader(self, `/ipfs/${options.formatsCid || IPLD_FORMATS_CID}`) - function createLoader (codec, path) { + // All known (non-default) IPLD formats + const IpldFormatLoaders = { + 'bitcoin-block': jsLoader( + '/ipld-bitcoin@0.1.9/index.min.js', + () => window.IpldBitcoin + ), + 'eth-account-snapshot': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethAccountSnapshot + ), + 'eth-block': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethBlock + ), + 'eth-block-list': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethBlockList + ), + 'eth-state-trie': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethStateTrie + ), + 'eth-storage-trie': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethStorageTrie + ), + 'eth-tx': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethTx + ), + 'eth-tx-trie': jsLoader( + '/ipld-ethereum@2.0.3/index.min.js', + () => window.IpldEthereum.ethTxTrie + ), + 'git-raw': jsLoader( + '/ipld-git@0.2.3/index.min.js', + () => window.IpldGit + ), + 'zcash-block': jsLoader( + '/ipld-zcash@0.1.6/index.min.js', + () => window.IpldZcash + ) + } + return (codec, callback) => { + if (IpldFormatLoaders[codec]) return IpldFormatLoaders[codec](callback) + callback(new Error(`Missing IPLD format "${codec}"`)) } +} - // All known (non-default) IPLD formats - const IpldFormatLoaders = { - 'bitcoin-block' (callback) { - if (IpldFormats['bitcoin-block']) { - return callback(null, IpldFormats['bitcoin-block']) - } +// Create a module loader for the passed root path e.g. /ipfs/QmHash +function createIpfsJsLoader (ipfs, rootPath) { + const Modules = {} - self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-bitcoin@0.1.9/index.min.js`, (err, data) => { - if (err) return callback(err) - const fn = new Function(data.toString()) // eslint-disable-line no-new-func - fn() - IpldFormats['bitcoin-block'] = window.IpldBitcoin - callback(null, IpldFormats['bitcoin-block']) - }) - }, - 'git-raw' (callback) { - if (IpldFormats['git-raw']) { - return callback(null, IpldFormats['git-raw']) + // Create a loader for the given path that will extract a JS object from + // the exports of the loaded module using getExport + return (path, getExport) => { + return callback => { + if (Modules[path]) { + switch (Modules[path].state) { + case 'loading': + return Modules[path].callbacks.push({ getExport, callback }) + case 'loaded': + return callback(null, getExport(Modules[path].exports)) + case 'error': + return callback(Modules[path].error) + } + return callback(new Error('unknown format load state')) } - console.log(`Loading from /ipfs/${IPLD_FORMATS_CID}/ipld-git@0.2.2/index.min.js`) + Modules[path] = { state: 'loading', callbacks: [{ getExport, callback }] } - self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-git@0.2.2/index.min.js`, (err, data) => { - if (err) return callback(err) - console.log('Loaded format git-raw') - const fn = new Function(data.toString()) // eslint-disable-line no-new-func - fn() - console.log('window.IpldGit', window.IpldGit) - IpldFormats['git-raw'] = window.IpldGit - callback(null, IpldFormats['git-raw']) - }) - }, - 'zcash-block' (callback) { - if (IpldFormats['zcash-block']) { - return callback(null, IpldFormats['zcash-block']) - } + ipfs.cat(`${rootPath}${path}`, (err, data) => { + if (err) { + Object.assign(Modules[path], { state: 'error', error: err }) + Modules[path].callbacks.forEach(({ callback }) => callback(err)) + Modules[path].callbacks = [] + return + } + + const exports = (new Function(data.toString()))() // eslint-disable-line no-new-func + Object.assign(Modules[path], { state: 'loaded', exports }) - self.cat(`/ipfs/${IPLD_FORMATS_CID}/ipld-zcash@0.1.6/index.min.js`, (err, data) => { - if (err) return callback(err) - const fn = new Function(data.toString()) // eslint-disable-line no-new-func - fn() - IpldFormats['zcash-block'] = window.IpldZcash - callback(null, IpldFormats['zcash-block']) + Modules[path].callbacks.forEach(({ getExport, callback }) => { + callback(null, getExport(exports)) + }) + Modules[path].callbacks = [] }) } } - - return (codec, callback) => { - if (IpldFormatLoaders[codec]) return IpldFormatLoaders[codec](callback) - callback(new Error(`Missing IPLD format "${codec}"`)) - } }