From afd2a2fba61cde3cfca89ea64266ceba86534cd4 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 4 Oct 2021 15:08:11 -0400 Subject: [PATCH 1/7] feat: add put / get methods --- src/index.js | 60 ++++++++++++++++++ ...mbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin | Bin 0 -> 166 bytes test/index.spec.js | 42 ++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin diff --git a/src/index.js b/src/index.js index a953663..91e95c1 100644 --- a/src/index.js +++ b/src/index.js @@ -129,6 +129,66 @@ class DelegatedContentRouting { }) log(`provide finished: ${key}`) } + + /** + * Stores a value in the backing key/value store of the delegated content router. + * This may fail if the delegated node's content routing implementation does not + * use a key/value store, or if the delegated operation fails. + * + * @param {Uint8Array|string} key - the key to store the value under + * @param {Uint8Array} value - a value to associate with the key. If a Uint8Array is given, it MUST contain valid UTF-8 text. + * @param {object} [options] + * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. + * @returns {Promise} + */ + async put (key, value, options = {}) { + const timeout = options.timeout || 3000 + const k = keyString(key) + log(`put value start: ${k}`) + await this._httpQueue.add(async () => { + await drain(this._client.dht.put(k, value, { timeout })) + }) + log(`put value finished: ${k}`) + } + + /** + * Fetches an value from the backing key/value store of the delegated content router. + * This may fail if the delegated node's content routing implementation does not + * use a key/value store, or if the delegated operation fails. + * + * @param {Uint8Array|string} key - the key to lookup. If a Uint8Array is given, it MUST contain valid UTF-8 text. + * @param {object} [options] + * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. + * @returns {Promise} the value for the given key. + */ + async get (key, options = {}) { + const timeout = options.timeout || 3000 + const k = keyString(key) + log(`get value start: ${k}`) + let value + await this._httpQueue.add(async () => { + value = await this._client.dht.get(k, { timeout }) + }) + log(`get value finished: ${k}`) + return value + } +} + +/** + * Helper to convert Uint8Array to UTF-8 text, or throw if key is invalid UTF-8 + * + * @param {Uint8Array|string} key + * @returns {string} + */ +const keyString = (key) => { + if (typeof key === 'string') { + return key + } + try { + return new TextDecoder().decode(key) + } catch (e) { + throw new Error(`Delegated routing supports only UTF-8 keys. Decoding error: ${e.message}`) + } } module.exports = DelegatedContentRouting diff --git a/test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin b/test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin new file mode 100644 index 0000000000000000000000000000000000000000..6ce6da746b65a6a14751ac48c9c2ee7e2cb218dc GIT binary patch literal 166 zcmV;X09pSEK`&`=W^*rMVP<(^Wocn)Z!=^wY;}1waC$dkWNBhyWM+DAZD??FY;AEj zVQp@5G p.id)).to.include(selfId.toB58String(), 'Did not include self node') }) }) + + describe('put', async () => { + it('should associate an IPNS record with a key', async () => { + const opts = delegateNode.apiAddr.toOptions() + const contentRouter = new DelegatedContentRouting(selfId, ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + await contentRouter.put(key, value) + + // check the delegate node to see if the value is retrievable + const fetched = await delegateNode.api.dht.get(key) + expect(fetched).to.deep.equal(value) + }) + }) + + describe('get', async () => { + it('should retrieve an IPNS record for a valid key', async () => { + const opts = delegateNode.apiAddr.toOptions() + const contentRouter = new DelegatedContentRouting(selfId, ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + // publish the record from the delegate node + await drain(delegateNode.api.dht.put(key, value)) + + // try to fetch it from the js node + const fetched = await contentRouter.get(key) + expect(fetched).to.deep.equal(value) + }) + }) }) From 53be33ed7bf1605630a8c7317a2f1d739b9de882 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 5 Oct 2021 12:37:02 -0400 Subject: [PATCH 2/7] define DelegatedValueStore class --- src/index.js | 60 --------------- src/value-store.js | 98 ++++++++++++++++++++++++ test/index.spec.js | 77 +------------------ test/test-utils.js | 44 +++++++++++ test/value-store.spec.js | 159 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 303 insertions(+), 135 deletions(-) create mode 100644 src/value-store.js create mode 100644 test/test-utils.js create mode 100644 test/value-store.spec.js diff --git a/src/index.js b/src/index.js index 91e95c1..a953663 100644 --- a/src/index.js +++ b/src/index.js @@ -129,66 +129,6 @@ class DelegatedContentRouting { }) log(`provide finished: ${key}`) } - - /** - * Stores a value in the backing key/value store of the delegated content router. - * This may fail if the delegated node's content routing implementation does not - * use a key/value store, or if the delegated operation fails. - * - * @param {Uint8Array|string} key - the key to store the value under - * @param {Uint8Array} value - a value to associate with the key. If a Uint8Array is given, it MUST contain valid UTF-8 text. - * @param {object} [options] - * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. - * @returns {Promise} - */ - async put (key, value, options = {}) { - const timeout = options.timeout || 3000 - const k = keyString(key) - log(`put value start: ${k}`) - await this._httpQueue.add(async () => { - await drain(this._client.dht.put(k, value, { timeout })) - }) - log(`put value finished: ${k}`) - } - - /** - * Fetches an value from the backing key/value store of the delegated content router. - * This may fail if the delegated node's content routing implementation does not - * use a key/value store, or if the delegated operation fails. - * - * @param {Uint8Array|string} key - the key to lookup. If a Uint8Array is given, it MUST contain valid UTF-8 text. - * @param {object} [options] - * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. - * @returns {Promise} the value for the given key. - */ - async get (key, options = {}) { - const timeout = options.timeout || 3000 - const k = keyString(key) - log(`get value start: ${k}`) - let value - await this._httpQueue.add(async () => { - value = await this._client.dht.get(k, { timeout }) - }) - log(`get value finished: ${k}`) - return value - } -} - -/** - * Helper to convert Uint8Array to UTF-8 text, or throw if key is invalid UTF-8 - * - * @param {Uint8Array|string} key - * @returns {string} - */ -const keyString = (key) => { - if (typeof key === 'string') { - return key - } - try { - return new TextDecoder().decode(key) - } catch (e) { - throw new Error(`Delegated routing supports only UTF-8 keys. Decoding error: ${e.message}`) - } } module.exports = DelegatedContentRouting diff --git a/src/value-store.js b/src/value-store.js new file mode 100644 index 0000000..674e5da --- /dev/null +++ b/src/value-store.js @@ -0,0 +1,98 @@ +'use strict' + +const debug = require('debug') +const drain = require('it-drain') +const { default: PQueue } = require('p-queue') + +const log = debug('libp2p-delegated-content-routing:value-store') +const CONCURRENT_HTTP_REQUESTS = 4 + +/** + * An implementation of the ValueStoreInterface using a delegated node. + */ +class DelegatedValueStore { + /** + * Create a new DelegatedValueStore instance. + * + * @param {object} client - an instance of the ipfs-http-client module + */ + constructor (client) { + if (client == null) { + throw new Error('missing ipfs http client') + } + + this._client = client + const concurrency = { concurrency: CONCURRENT_HTTP_REQUESTS } + this._httpQueue = new PQueue(concurrency) + + const { + protocol, + host, + port + } = client.getEndpointConfig() + + log(`enabled DelegatedValueStore via ${protocol}://${host}:${port}`) + } + + /** + * Stores a value in the backing key/value store of the delegated content router. + * This may fail if the delegated node's content routing implementation does not + * use a key/value store, or if the delegated operation fails. + * + * @param {Uint8Array|string} key - the key to store the value under + * @param {Uint8Array} value - a value to associate with the key. If a Uint8Array is given, it MUST contain valid UTF-8 text. + * @param {object} [options] + * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. + * @returns {Promise} + */ + async put (key, value, options = {}) { + const timeout = options.timeout || 3000 + const k = keyString(key) + log(`put value start: ${k}`) + await this._httpQueue.add(async () => { + await drain(this._client.dht.put(k, value, { timeout })) + }) + log(`put value finished: ${k}`) + } + + /** + * Fetches an value from the backing key/value store of the delegated content router. + * This may fail if the delegated node's content routing implementation does not + * use a key/value store, or if the delegated operation fails. + * + * @param {Uint8Array|string} key - the key to lookup. If a Uint8Array is given, it MUST contain valid UTF-8 text. + * @param {object} [options] + * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. + * @returns {Promise} the value for the given key. + */ + async get (key, options = {}) { + const timeout = options.timeout || 3000 + const k = keyString(key) + log(`get value start: ${k}`) + let value + await this._httpQueue.add(async () => { + value = await this._client.dht.get(k, { timeout }) + }) + log(`get value finished: ${k}`) + return value + } +} + +/** + * Helper to convert Uint8Array to UTF-8 text, or throw if key is invalid UTF-8 + * + * @param {Uint8Array|string} key + * @returns {string} + */ +const keyString = (key) => { + if (typeof key === 'string') { + return key + } + try { + return new TextDecoder().decode(key) + } catch (e) { + throw new Error(`Delegated routing supports only UTF-8 keys. Decoding error: ${e.message}`) + } +} + +module.exports = DelegatedValueStore diff --git a/test/index.spec.js b/test/index.spec.js index daf175d..42c40b5 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -1,49 +1,17 @@ /* eslint-env mocha */ 'use strict' -const loadFixture = require('aegir/utils/fixtures') const { expect } = require('aegir/utils/chai') -const { createFactory } = require('ipfsd-ctl') const ipfsHttpClient = require('ipfs-http-client') const { CID } = ipfsHttpClient const PeerId = require('peer-id') const all = require('it-all') const drain = require('it-drain') -const { isNode } = require('ipfs-utils/src/env') const uint8ArrayFromString = require('uint8arrays/from-string') -const factory = createFactory({ - type: 'go', - ipfsHttpModule: require('ipfs-http-client'), - ipfsBin: isNode ? require('go-ipfs').path() : undefined, - test: true, - endpoint: 'http://localhost:57483' -}) +const { spawnNode, cleanupNodeFactory } = require('./test-utils') const DelegatedContentRouting = require('../src') -async function spawnNode (bootstrap = []) { - const node = await factory.spawn({ - // Lock down the nodes so testing can be deterministic - ipfsOptions: { - config: { - Bootstrap: bootstrap, - Discovery: { - MDNS: { - Enabled: false - } - } - } - } - }) - - const id = await node.api.id() - - return { - node, - id - } -} - describe('DelegatedContentRouting', function () { this.timeout(20 * 1000) // we're spawning daemons, give ci some time @@ -70,7 +38,7 @@ describe('DelegatedContentRouting', function () { }) after(() => { - return factory.clean() + return cleanupNodeFactory() }) describe('create', () => { @@ -184,45 +152,4 @@ describe('DelegatedContentRouting', function () { expect(providers.map((p) => p.id)).to.include(selfId.toB58String(), 'Did not include self node') }) }) - - describe('put', async () => { - it('should associate an IPNS record with a key', async () => { - const opts = delegateNode.apiAddr.toOptions() - const contentRouter = new DelegatedContentRouting(selfId, ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' - const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') - - await contentRouter.put(key, value) - - // check the delegate node to see if the value is retrievable - const fetched = await delegateNode.api.dht.get(key) - expect(fetched).to.deep.equal(value) - }) - }) - - describe('get', async () => { - it('should retrieve an IPNS record for a valid key', async () => { - const opts = delegateNode.apiAddr.toOptions() - const contentRouter = new DelegatedContentRouting(selfId, ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' - const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') - - // publish the record from the delegate node - await drain(delegateNode.api.dht.put(key, value)) - - // try to fetch it from the js node - const fetched = await contentRouter.get(key) - expect(fetched).to.deep.equal(value) - }) - }) }) diff --git a/test/test-utils.js b/test/test-utils.js new file mode 100644 index 0000000..839ceea --- /dev/null +++ b/test/test-utils.js @@ -0,0 +1,44 @@ +'use strict' + +const { createFactory } = require('ipfsd-ctl') +const { isNode } = require('ipfs-utils/src/env') + +const factory = createFactory({ + type: 'go', + ipfsHttpModule: require('ipfs-http-client'), + ipfsBin: isNode ? require('go-ipfs').path() : undefined, + test: true, + endpoint: 'http://localhost:57483' +}) + +async function spawnNode (bootstrap = []) { + const node = await factory.spawn({ + // Lock down the nodes so testing can be deterministic + ipfsOptions: { + config: { + Bootstrap: bootstrap, + Discovery: { + MDNS: { + Enabled: false + } + } + } + } + }) + + const id = await node.api.id() + + return { + node, + id + } +} + +function cleanupNodeFactory () { + return factory.clean() +} + +module.exports = { + spawnNode, + cleanupNodeFactory +} diff --git a/test/value-store.spec.js b/test/value-store.spec.js new file mode 100644 index 0000000..61577ef --- /dev/null +++ b/test/value-store.spec.js @@ -0,0 +1,159 @@ +/* eslint-env mocha */ +'use strict' + +const loadFixture = require('aegir/utils/fixtures') +const { expect } = require('aegir/utils/chai') +const ipfsHttpClient = require('ipfs-http-client') +const drain = require('it-drain') +const { spawnNode, cleanupNodeFactory } = require('./test-utils') + +const DelegatedValueStore = require('../src/value-store') + +describe('DelegatedValueStore', function () { + this.timeout(20 * 1000) // we're spawning daemons, give ci some time + + let delegateNode + + before(async () => { + // Spawn a "Boostrap" node that doesnt connect to anything + const bootstrap = await spawnNode() + const bootstrapId = bootstrap.id + + // Spawn the delegate node and bootstrap the bootstrapper node + const delegate = await spawnNode(bootstrapId.addresses) + delegateNode = delegate.node + }) + + after(() => { + return cleanupNodeFactory() + }) + + describe('create', () => { + it('should require ipfs http client', () => { + expect(() => new DelegatedValueStore()).to.throw() + }) + + it('should accept an http api client instance at construction time', () => { + const client = ipfsHttpClient.create({ + protocol: 'http', + port: 8000, + host: 'localhost' + }) + const valueStore = new DelegatedValueStore(client) + + expect(valueStore).to.have.property('_client') + .that.has.property('getEndpointConfig') + .that.is.a('function') + + expect(valueStore._client.getEndpointConfig()).to.deep.include({ + protocol: 'http:', + port: '8000', + host: 'localhost' + }) + }) + }) + + describe('put', async () => { + it('should associate an IPNS record with a key', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + await valueStore.put(key, value) + + // check the delegate node to see if the value is retrievable + const fetched = await delegateNode.api.dht.get(key) + expect(fetched).to.deep.equal(value) + }) + + it('should accept UTF-8 keys as Uint8Arrays', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const keyStr = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + await valueStore.put(new TextEncoder().encode(keyStr), value) + + // check the delegate node to see if the value is retrievable + const fetched = await delegateNode.api.dht.get(keyStr) + expect(fetched).to.deep.equal(value) + }) + + it('should fail if keys are not valid UTF-8', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = new TextEncoder([0xff]) + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + expect(valueStore.put(key, value)).to.be.rejectedWith('UTF-8') + }) + }) + + describe('get', async () => { + it('should retrieve an IPNS record for a valid key', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + // publish the record from the delegate node + await drain(delegateNode.api.dht.put(key, value)) + + // try to fetch it from the js node + const fetched = await valueStore.get(key) + expect(fetched).to.deep.equal(value) + }) + + it('should accept UTF-8 keys as Uint8Arrays', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const keyStr = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') + + // publish the record from the delegate node + await drain(delegateNode.api.dht.put(keyStr, value)) + + // try to fetch it from the js node + const fetched = await valueStore.get(new TextEncoder().encode(keyStr)) + expect(fetched).to.deep.equal(value) + }) + + it('should fail if key is not valid UTF-8', async () => { + const opts = delegateNode.apiAddr.toOptions() + const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + protocol: 'http', + port: opts.port, + host: opts.host + })) + + const key = new TextEncoder([0xff]) + expect(valueStore.get(key)).to.be.rejectedWith('UTF-8') + }) + }) +}) From 2d3dfd8a7ab85796cb76a0b5d9f6658f2b87f04f Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Oct 2021 12:17:20 -0400 Subject: [PATCH 3/7] update ipfs-http-client dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ac3d387..87daf9f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "devDependencies": { "aegir": "^33.0.0", "go-ipfs": "^0.8.0", - "ipfs-http-client": "^50.1.0", + "ipfs-http-client": "^52.0.2", "ipfs-utils": "^8.1.2", "ipfsd-ctl": "^8.0.2", "it-all": "^1.0.0", From a27ab398b01bb4fa6c020f39a5b3bcc96d49f830 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Oct 2021 12:41:14 -0400 Subject: [PATCH 4/7] use binary keys to satisfy new client reqs --- src/value-store.js | 35 +++++---------------- test/index.spec.js | 2 +- test/value-store.spec.js | 67 ++-------------------------------------- 3 files changed, 11 insertions(+), 93 deletions(-) diff --git a/src/value-store.js b/src/value-store.js index 674e5da..32edab3 100644 --- a/src/value-store.js +++ b/src/value-store.js @@ -39,20 +39,19 @@ class DelegatedValueStore { * This may fail if the delegated node's content routing implementation does not * use a key/value store, or if the delegated operation fails. * - * @param {Uint8Array|string} key - the key to store the value under - * @param {Uint8Array} value - a value to associate with the key. If a Uint8Array is given, it MUST contain valid UTF-8 text. + * @param {Uint8Array} key - the key to store the value under + * @param {Uint8Array} value - a value to associate with the key. * @param {object} [options] * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. * @returns {Promise} */ async put (key, value, options = {}) { const timeout = options.timeout || 3000 - const k = keyString(key) - log(`put value start: ${k}`) + log(`put value start: ${key}`) await this._httpQueue.add(async () => { - await drain(this._client.dht.put(k, value, { timeout })) + await drain(this._client.dht.put(key, value, { timeout })) }) - log(`put value finished: ${k}`) + log(`put value finished: ${key}`) } /** @@ -67,32 +66,14 @@ class DelegatedValueStore { */ async get (key, options = {}) { const timeout = options.timeout || 3000 - const k = keyString(key) - log(`get value start: ${k}`) + log(`get value start: ${key}`) let value await this._httpQueue.add(async () => { - value = await this._client.dht.get(k, { timeout }) + value = await this._client.dht.get(key, { timeout }) }) - log(`get value finished: ${k}`) + log(`get value finished: ${key}`) return value } } -/** - * Helper to convert Uint8Array to UTF-8 text, or throw if key is invalid UTF-8 - * - * @param {Uint8Array|string} key - * @returns {string} - */ -const keyString = (key) => { - if (typeof key === 'string') { - return key - } - try { - return new TextDecoder().decode(key) - } catch (e) { - throw new Error(`Delegated routing supports only UTF-8 keys. Decoding error: ${e.message}`) - } -} - module.exports = DelegatedValueStore diff --git a/test/index.spec.js b/test/index.spec.js index 42c40b5..98cfec1 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -72,7 +72,7 @@ describe('DelegatedContentRouting', function () { describe('findProviders', () => { const data = uint8ArrayFromString('some data') - const cid = new CID('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') // 'some data' + const cid = CID.parse('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') // 'some data' before('register providers', async () => { await Promise.all([ diff --git a/test/value-store.spec.js b/test/value-store.spec.js index 61577ef..077d4f9 100644 --- a/test/value-store.spec.js +++ b/test/value-store.spec.js @@ -62,7 +62,7 @@ describe('DelegatedValueStore', function () { host: opts.host })) - const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const key = new TextEncoder().encode('/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu') const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') await valueStore.put(key, value) @@ -71,38 +71,6 @@ describe('DelegatedValueStore', function () { const fetched = await delegateNode.api.dht.get(key) expect(fetched).to.deep.equal(value) }) - - it('should accept UTF-8 keys as Uint8Arrays', async () => { - const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const keyStr = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' - const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') - - await valueStore.put(new TextEncoder().encode(keyStr), value) - - // check the delegate node to see if the value is retrievable - const fetched = await delegateNode.api.dht.get(keyStr) - expect(fetched).to.deep.equal(value) - }) - - it('should fail if keys are not valid UTF-8', async () => { - const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const key = new TextEncoder([0xff]) - const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') - - expect(valueStore.put(key, value)).to.be.rejectedWith('UTF-8') - }) }) describe('get', async () => { @@ -114,7 +82,7 @@ describe('DelegatedValueStore', function () { host: opts.host })) - const key = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' + const key = new TextEncoder().encode('/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu') const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') // publish the record from the delegate node @@ -124,36 +92,5 @@ describe('DelegatedValueStore', function () { const fetched = await valueStore.get(key) expect(fetched).to.deep.equal(value) }) - - it('should accept UTF-8 keys as Uint8Arrays', async () => { - const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const keyStr = '/ipns/k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu' - const value = loadFixture('test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin') - - // publish the record from the delegate node - await drain(delegateNode.api.dht.put(keyStr, value)) - - // try to fetch it from the js node - const fetched = await valueStore.get(new TextEncoder().encode(keyStr)) - expect(fetched).to.deep.equal(value) - }) - - it('should fail if key is not valid UTF-8', async () => { - const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ - protocol: 'http', - port: opts.port, - host: opts.host - })) - - const key = new TextEncoder([0xff]) - expect(valueStore.get(key)).to.be.rejectedWith('UTF-8') - }) }) }) From 50cb95024196777e82f0e03d5e9a4b1acfb6aac3 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Oct 2021 12:41:01 -0400 Subject: [PATCH 5/7] use ipns key with 100 year expiry for test fixture --- ...agmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin | Bin 166 -> 166 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin b/test/fixtures/ipns-k51qzi5uqu5dgg9b8xoi0yagmbl6iyu0k1epa4hew8jm3z9c7zzmkkl1t4hihu.bin index 6ce6da746b65a6a14751ac48c9c2ee7e2cb218dc..0bf1baeeb380d981b3464381ba970f9c6d02668e 100644 GIT binary patch delta 104 zcmV-u0GI!!0j2?vMPXT@&#ZNr)EG?390I`V{<}GByJVx4qd@i1ygCra$9m>W8wwc< zGc_$SF)c7QR4_O?Gc-CdGcGwd KG&M3eS||a}^eDgp delta 104 zcmV-u0GI!!0j2?vMPc`NJZdDipBdz~8z4~Uqwq8*ygJ9%LyOud6SWz86hq*Lsn-${ zOKPM^^t6TjL9S;^Ae;{m!1LR{$6QA{#O=GBGVNFfA}OR53U@Gd4OhGcGbT KGC46aS||aDnkfwc From 7636f7fd822328d34f5ba05c65c9a464678531bb Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Oct 2021 14:21:55 -0400 Subject: [PATCH 6/7] return GetValueResult from get --- src/value-store.js | 27 ++++++++++++++++++++++----- test/value-store.spec.js | 18 ++++++++++++------ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/value-store.js b/src/value-store.js index 32edab3..b62b6a6 100644 --- a/src/value-store.js +++ b/src/value-store.js @@ -7,6 +7,15 @@ const { default: PQueue } = require('p-queue') const log = debug('libp2p-delegated-content-routing:value-store') const CONCURRENT_HTTP_REQUESTS = 4 + +/** + * @typedef {{import('peer-id')}.PeerId} PeerId + * + * @typedef {object} GetValueResult + * @property {PeerId} from + * @property {Uint8Array} val + */ + /** * An implementation of the ValueStoreInterface using a delegated node. */ @@ -14,13 +23,19 @@ class DelegatedValueStore { /** * Create a new DelegatedValueStore instance. * + * @param {PeerId} delegateId - the peer id of the delegate node * @param {object} client - an instance of the ipfs-http-client module */ - constructor (client) { + constructor (delegateId, client) { + if (delegateId == null) { + throw new Error('missing delegate peer id') + } + if (client == null) { throw new Error('missing ipfs http client') } + this._delegateId = delegateId this._client = client const concurrency = { concurrency: CONCURRENT_HTTP_REQUESTS } this._httpQueue = new PQueue(concurrency) @@ -62,17 +77,19 @@ class DelegatedValueStore { * @param {Uint8Array|string} key - the key to lookup. If a Uint8Array is given, it MUST contain valid UTF-8 text. * @param {object} [options] * @param {number} [options.timeout] - a timeout in ms. Defaults to 30s. - * @returns {Promise} the value for the given key. + * @returns {Promise} the value for the given key. */ async get (key, options = {}) { const timeout = options.timeout || 3000 log(`get value start: ${key}`) - let value + let val await this._httpQueue.add(async () => { - value = await this._client.dht.get(key, { timeout }) + val = await this._client.dht.get(key, { timeout }) }) log(`get value finished: ${key}`) - return value + + const from = this._delegateId + return { from, val } } } diff --git a/test/value-store.spec.js b/test/value-store.spec.js index 077d4f9..60ac68d 100644 --- a/test/value-store.spec.js +++ b/test/value-store.spec.js @@ -13,6 +13,7 @@ describe('DelegatedValueStore', function () { this.timeout(20 * 1000) // we're spawning daemons, give ci some time let delegateNode + let delegateId before(async () => { // Spawn a "Boostrap" node that doesnt connect to anything @@ -22,6 +23,7 @@ describe('DelegatedValueStore', function () { // Spawn the delegate node and bootstrap the bootstrapper node const delegate = await spawnNode(bootstrapId.addresses) delegateNode = delegate.node + delegateId = await delegateNode.api.id() }) after(() => { @@ -29,9 +31,12 @@ describe('DelegatedValueStore', function () { }) describe('create', () => { - it('should require ipfs http client', () => { + it('should require the peer id of the delegate node', () => { expect(() => new DelegatedValueStore()).to.throw() }) + it('should require ipfs http client', () => { + expect(() => new DelegatedValueStore(delegateId)).to.throw() + }) it('should accept an http api client instance at construction time', () => { const client = ipfsHttpClient.create({ @@ -39,7 +44,7 @@ describe('DelegatedValueStore', function () { port: 8000, host: 'localhost' }) - const valueStore = new DelegatedValueStore(client) + const valueStore = new DelegatedValueStore(delegateId, client) expect(valueStore).to.have.property('_client') .that.has.property('getEndpointConfig') @@ -56,7 +61,7 @@ describe('DelegatedValueStore', function () { describe('put', async () => { it('should associate an IPNS record with a key', async () => { const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + const valueStore = new DelegatedValueStore(delegateId, ipfsHttpClient.create({ protocol: 'http', port: opts.port, host: opts.host @@ -76,7 +81,7 @@ describe('DelegatedValueStore', function () { describe('get', async () => { it('should retrieve an IPNS record for a valid key', async () => { const opts = delegateNode.apiAddr.toOptions() - const valueStore = new DelegatedValueStore(ipfsHttpClient.create({ + const valueStore = new DelegatedValueStore(delegateId, ipfsHttpClient.create({ protocol: 'http', port: opts.port, host: opts.host @@ -89,8 +94,9 @@ describe('DelegatedValueStore', function () { await drain(delegateNode.api.dht.put(key, value)) // try to fetch it from the js node - const fetched = await valueStore.get(key) - expect(fetched).to.deep.equal(value) + const result = await valueStore.get(key) + expect(result.from).to.deep.equal(delegateId) + expect(result.val).to.deep.equal(value) }) }) }) From 362cd00988e717f6fc49b0a1f2fa7bbaabbfcc53 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 7 Oct 2021 14:23:58 -0400 Subject: [PATCH 7/7] fix lint issues --- src/value-store.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/value-store.js b/src/value-store.js index b62b6a6..efdbf24 100644 --- a/src/value-store.js +++ b/src/value-store.js @@ -7,10 +7,9 @@ const { default: PQueue } = require('p-queue') const log = debug('libp2p-delegated-content-routing:value-store') const CONCURRENT_HTTP_REQUESTS = 4 - /** * @typedef {{import('peer-id')}.PeerId} PeerId - * + * * @typedef {object} GetValueResult * @property {PeerId} from * @property {Uint8Array} val @@ -87,7 +86,7 @@ class DelegatedValueStore { val = await this._client.dht.get(key, { timeout }) }) log(`get value finished: ${key}`) - + const from = this._delegateId return { from, val } }