Merge pull request #1 from ipfs/feat/chunker-fixed-size

Fixed Size Chunker

package.json

@@ -0,0 +1,34 @@
+{
+  "name": "ipfs-data-importing",
+  "version": "0.0.0",
+  "description": "JavaScript implementation of the layout and chunking mechanisms used by IPFS",
+  "main": "src/index.js",
+  "scripts": {
+    "lint": "standard",
+    "coverage": "istanbul cover --print both -- _mocha tests/test-*.js",
+    "test": "mocha tests/test-*.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/diasdavid/js-ipfs-data-importing.git"
+  },
+  "keywords": [
+    "IPFS"
+  ],
+  "author": "David Dias <daviddias@ipfs.io>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/diasdavid/js-ipfs-data-importing/issues"
+  },
+  "homepage": "https://github.com/diasdavid/js-ipfs-data-importing#readme",
+  "devDependencies": {
+    "chai": "^3.4.1",
+    "istanbul": "^0.4.1",
+    "mocha": "^2.3.4",
+    "pre-commit": "^1.1.2",
+    "standard": "^5.4.1"
+  },
+  "dependencies": {
+    "debug": "^2.2.0"
+  }
+}

src/chunker-fixed-size.js

@@ -0,0 +1,45 @@
+var through2 = require('through2')
+
+exports = module.exports = FixedSizeChunker
+
+// The difference of this chunker compared to other fixed size chunkers
+// available, is that it doesn't add padding the last chunk
+
+function FixedSizeChunker (size) {
+  var stream = through2(transform, flush)
+
+  var buf = new Buffer(0)
+
+  function transform (chunk, enc, cb) {
+    var that = this
+
+    buf = Buffer.concat([buf, chunk])
+
+    if (buf.length >= size) {
+      slice()
+    }
+
+    function slice () {
+      var chunk = new Buffer(size)
+      var newBuf = new Buffer(buf.length - size)
+      buf.copy(chunk, 0, 0, size - 1)
+      buf.copy(newBuf, 0, size - 1, buf.length - size)
+      buf = newBuf
+      that.push(chunk)
+
+      if (buf.length >= size) {
+        return slice()
+      }
+    }
+
+    cb()
+  }
+
+  function flush (cb) {
+    // last chunk
+    this.push(buf)
+    cb()
+  }
+
+  return stream
+}

src/index.js

@@ -0,0 +1,27 @@
+var debug = require('debug')
+var log = debug('importer')
+log.err = debug('importer:error')
+var fs = require('fs')
+
+exports = module.exports
+
+// Use a layout + chunkers to convert a directory (or file) to the layout format
+exports.import = function (options, callback) {
+  // options.store -> where to write stuff (typically js-ipfs-repo.datastore, which impls blob-store)
+  // options.path -> where to
+  // options.recursive -> follow dirs
+  // options.chunkers -> obj with chunkers to each type of data, { default: dumb-chunker }
+
+  var pathStats = fs.statSync(options.path)
+  if (pathStats.isFile()) {
+
+
+  } else if (pathStats.isDir() && options.recursive) {
+
+  } else {
+    return callback(new Error('recursive must be true to add a directory'))
+  }
+}
+
+exports.export = function () {
+}

src/layout-merkledag.js

@@ -0,0 +1 @@
+// protobufs

tests/test-data/200Bytes.txt

@@ -0,0 +1,4 @@
+�wx���xM��{
+D���zH/&^����RS���/��v,��R
+�=��N���g~���pf1�\[�>�%��U�1�@Q���׀2&m6�q���Q؁��]��|���!�KE�~J֕읝�o�j��b�n3�eT�)D+;s
+컓��:Ty!c�3����\*���T7��E?[��Pv}��A+�c�x�~�e�

tests/test-fixed-size-chunker.js

@@ -0,0 +1,61 @@
+/* globals describe, it */
+
+var FixedSizeChunker = require('./../src/chunker-fixed-size')
+var fs = require('fs')
+var stream = require('stream')
+var expect = require('chai').expect
+
+describe('chunker: fixed size', function () {
+  it('256 Bytes chunks', function (done) {
+    var writable = new stream.Writable({
+      write: function (chunk, encoding, next) {
+        expect(chunk.length).to.equal(256)
+        next()
+      }
+    })
+
+    fs.createReadStream(__dirname + '/test-data/1MiB.txt')
+      .pipe(FixedSizeChunker(256))
+      .pipe(writable)
+
+    writable.on('finish', done)
+  })
+
+  it('256 KiB chunks', function (done) {
+    var KiB256 = 262144
+    var writable = new stream.Writable({
+      write: function (chunk, encoding, next) {
+        expect(chunk.length).to.equal(KiB256)
+        next()
+      }
+    })
+
+    fs.createReadStream(__dirname + '/test-data/1MiB.txt')
+      .pipe(FixedSizeChunker(KiB256))
+      .pipe(writable)
+
+    writable.on('finish', done)
+  })
+
+  it('256 KiB chunks of non scalar filesize', function (done) {
+    var counter = 0
+    var KiB256 = 262144
+    var writable = new stream.Writable({
+      write: function (chunk, encoding, next) {
+        if (chunk.length < KiB256) {
+          expect(counter).to.be.below(2)
+          counter += 1
+          return next()
+        }
+        expect(chunk.length).to.equal(KiB256)
+        next()
+      }
+    })
+
+    fs.createReadStream(__dirname + '/test-data/1.2MiB.txt')
+      .pipe(FixedSizeChunker(KiB256))
+      .pipe(writable)
+
+    writable.on('finish', done)
+  })
+})