fix: remove node buffers (#44)

- Updates to the latest changes from interface-datastore and datastore-core - Uses it-glob to not buffer datastore paths during queries BREAKING CHANGE: only uses Uint8Arrays internally
ipfs · Jul 29, 2020 · 887b762 · 887b762
1 parent 9963b95
commit 887b762
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 27 deletions.
diff --git a/package.json b/package.json
@@ -5,10 +5,7 @@
   "leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
   "main": "src/index.js",
   "scripts": {
-    "test": "aegir test",
-    "test:node": "aegir test -t node",
-    "test:browser": "aegir test -t browser",
-    "test:webworker": "aegir test -t webworker",
+    "test": "aegir test -t node",
     "build": "aegir build",
     "lint": "aegir lint",
     "release": "aegir release",
@@ -35,19 +32,20 @@
   },
   "homepage": "https://github.com/ipfs/js-datastore-fs#readme",
   "dependencies": {
-    "datastore-core": "^1.1.0",
+    "datastore-core": "^2.0.0",
     "fast-write-atomic": "^0.2.0",
-    "glob": "^7.1.3",
-    "interface-datastore": "^1.0.2",
+    "interface-datastore": "^2.0.0",
+    "it-glob": "0.0.8",
     "mkdirp": "^1.0.4"
   },
   "devDependencies": {
-    "aegir": "^22.0.0",
+    "aegir": "^25.0.0",
     "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
-    "cids": "~0.8.0",
+    "cids": "^0.8.3",
     "detect-node": "^2.0.4",
     "dirty-chai": "^2.0.1",
+    "ipfs-utils": "^2.3.1",
     "memdown": "^5.1.0",
     "rimraf": "^3.0.2"
   },

diff --git a/src/index.js b/src/index.js
@@ -1,7 +1,7 @@
 'use strict'
 
 const fs = require('fs')
-const glob = require('glob')
+const glob = require('it-glob')
 const mkdirp = require('mkdirp')
 const promisify = require('util').promisify
 const writeAtomic = promisify(require('fast-write-atomic'))
@@ -117,7 +117,7 @@ class FsDatastore extends Adapter {
    * Write to the file system without extension.
    *
    * @param {Key} key
-   * @param {Buffer} val
+   * @param {Uint8Array} val
    * @returns {Promise<void>}
    */
   async putRaw (key, val) {
@@ -131,11 +131,12 @@ class FsDatastore extends Adapter {
    * Store the given value under the key
    *
    * @param {Key} key
-   * @param {Buffer} val
+   * @param {Uint8Array} val
    * @returns {Promise<void>}
    */
   async put (key, val) {
     const parts = this._encode(key)
+
     try {
       await mkdirp(parts.dir, { fs: fs })
       await writeFile(parts.file, val)
@@ -148,7 +149,7 @@ class FsDatastore extends Adapter {
    * Read from the file system without extension.
    *
    * @param {Key} key
-   * @returns {Promise<Buffer>}
+   * @returns {Promise<Uint8Array>}
    */
   async getRaw (key) {
     const parts = this._encode(key)
@@ -167,7 +168,7 @@ class FsDatastore extends Adapter {
    * Read from the file system.
    *
    * @param {Key} key
-   * @returns {Promise<Buffer>}
+   * @returns {Promise<Uint8Array>}
    */
   async get (key) {
     const parts = this._encode(key)
@@ -216,22 +217,35 @@ class FsDatastore extends Adapter {
   }
 
   async * _all (q) { // eslint-disable-line require-await
-    // glob expects a POSIX path
-    const prefix = q.prefix || '**'
-    const pattern = path
-      .join(this.path, prefix, '*' + this.opts.extension)
+    let prefix = q.prefix || '**'
+
+    // strip leading slashes
+    prefix = prefix.replace(/^\/+/, '')
+
+    const pattern = `${prefix}/*${this.opts.extension}`
       .split(path.sep)
       .join('/')
-    const files = glob.sync(pattern)
+    const files = glob(this.path, pattern, {
+      absolute: true
+    })
 
     if (!q.keysOnly) {
-      yield * map(files, async (f) => {
-        const buf = await fsReadFile(f)
-        return {
-          key: this._decode(f),
-          value: buf
+      for await (const file of files) {
+        try {
+          const buf = await fsReadFile(file)
+
+          yield {
+            key: this._decode(file),
+            value: buf
+          }
+        } catch (err) {
+          // if keys are removed from the datastore while the query is
+          // running, we may encounter missing files.
+          if (err.code !== 'ENOENT') {
+            throw err
+          }
         }
-      })
+      }
     } else {
       yield * map(files, f => ({ key: this._decode(f) }))
     }

diff --git a/test/index.spec.js b/test/index.spec.js
@@ -16,6 +16,8 @@ const utils = require('interface-datastore').utils
 const ShardingStore = require('datastore-core').ShardingDatastore
 const sh = require('datastore-core').shard
 const isNode = require('detect-node')
+const TextEncoder = require('ipfs-utils/src/text-encoder')
+const utf8Encoder = new TextEncoder('utf8')
 
 const FsStore = require('../src')
 
@@ -87,7 +89,7 @@ describe('FsDatastore', () => {
     const fs = new FsStore(dir)
     const key = new Key('1234')
 
-    await fs.put(key, Buffer.from([0, 1, 2, 3]))
+    await fs.put(key, Uint8Array.from([0, 1, 2, 3]))
     await fs.delete(key)
 
     try {
@@ -181,7 +183,7 @@ describe('FsDatastore', () => {
     const dir = utils.tmpdir()
     const fstore = new FsStore(dir)
     const key = new Key('CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY')
-    const value = Buffer.from('Hello world')
+    const value = utf8Encoder.encode('Hello world')
 
     await Promise.all(
       new Array(100).fill(0).map(() => fstore.put(key, value))