diff --git a/lib/search.js b/lib/search.js index 3987be135c9ae..3c59f8b43d15b 100644 --- a/lib/search.js +++ b/lib/search.js @@ -2,14 +2,16 @@ module.exports = exports = search -var npm = require('./npm.js') -var allPackageSearch = require('./search/all-package-search') -var esearch = require('./search/esearch.js') -var formatPackageStream = require('./search/format-package-stream.js') -var usage = require('./utils/usage') -var output = require('./utils/output.js') -var log = require('npmlog') -var ms = require('mississippi') +const npm = require('./npm.js') +const allPackageSearch = require('./search/all-package-search') +const figgyPudding = require('figgy-pudding') +const formatPackageStream = require('./search/format-package-stream.js') +const libSearch = require('libnpm/search') +const log = require('npmlog') +const ms = require('mississippi') +const npmConfig = require('./config/figgy-config.js') +const output = require('./utils/output.js') +const usage = require('./utils/usage') search.usage = usage( 'search', @@ -20,46 +22,50 @@ search.completion = function (opts, cb) { cb(null, []) } +const SearchOpts = figgyPudding({ + description: {}, + exclude: {}, + include: {}, + limit: {}, + log: {}, + staleness: {}, + unicode: {} +}) + function search (args, cb) { - var searchOpts = { + const opts = SearchOpts(npmConfig()).concat({ description: npm.config.get('description'), exclude: prepareExcludes(npm.config.get('searchexclude')), include: prepareIncludes(args, npm.config.get('searchopts')), - limit: npm.config.get('searchlimit'), + limit: npm.config.get('searchlimit') || 20, log: log, staleness: npm.config.get('searchstaleness'), unicode: npm.config.get('unicode') - } - - if (searchOpts.include.length === 0) { + }) + if (opts.include.length === 0) { return cb(new Error('search must be called with arguments')) } // Used later to figure out whether we had any packages go out - var anyOutput = false + let anyOutput = false - var entriesStream = ms.through.obj() + const entriesStream = ms.through.obj() - var esearchWritten = false - esearch(searchOpts).on('data', function (pkg) { + let esearchWritten = false + libSearch.stream(opts.include, opts).on('data', pkg => { entriesStream.write(pkg) !esearchWritten && (esearchWritten = true) - }).on('error', function (e) { + }).on('error', err => { if (esearchWritten) { // If esearch errored after already starting output, we can't fall back. - return entriesStream.emit('error', e) + return entriesStream.emit('error', err) } log.warn('search', 'fast search endpoint errored. Using old search.') - allPackageSearch(searchOpts).on('data', function (pkg) { - entriesStream.write(pkg) - }).on('error', function (e) { - entriesStream.emit('error', e) - }).on('end', function () { - entriesStream.end() - }) - }).on('end', function () { - entriesStream.end() - }) + allPackageSearch(opts) + .on('data', pkg => entriesStream.write(pkg)) + .on('error', err => entriesStream.emit('error', err)) + .on('end', () => entriesStream.end()) + }).on('end', () => entriesStream.end()) // Grab a configured output stream that will spit out packages in the // desired format. @@ -71,14 +77,14 @@ function search (args, cb) { parseable: npm.config.get('parseable'), color: npm.color }) - outputStream.on('data', function (chunk) { + outputStream.on('data', chunk => { if (!anyOutput) { anyOutput = true } output(chunk.toString('utf8')) }) log.silly('search', 'searching packages') - ms.pipe(entriesStream, outputStream, function (er) { - if (er) return cb(er) + ms.pipe(entriesStream, outputStream, err => { + if (err) return cb(err) if (!anyOutput && !npm.config.get('json') && !npm.config.get('parseable')) { output('No matches found for ' + (args.map(JSON.stringify).join(' '))) } diff --git a/lib/search/all-package-metadata.js b/lib/search/all-package-metadata.js index 5a27bdbcee658..5883def5c72e3 100644 --- a/lib/search/all-package-metadata.js +++ b/lib/search/all-package-metadata.js @@ -1,21 +1,28 @@ 'use strict' -var fs = require('graceful-fs') -var path = require('path') -var mkdir = require('mkdirp') -var chownr = require('chownr') -var npm = require('../npm.js') -var log = require('npmlog') -var cacheFile = require('npm-cache-filename') -var correctMkdir = require('../utils/correct-mkdir.js') -var mapToRegistry = require('../utils/map-to-registry.js') -var jsonstream = require('JSONStream') -var writeStreamAtomic = require('fs-write-stream-atomic') -var ms = require('mississippi') -var sortedUnionStream = require('sorted-union-stream') -var once = require('once') -var gunzip = require('../utils/gunzip-maybe') +const BB = require('bluebird') +const cacheFile = require('npm-cache-filename') +const chownr = BB.promisify(require('chownr')) +const correctMkdir = BB.promisify(require('../utils/correct-mkdir.js')) +const figgyPudding = require('figgy-pudding') +const fs = require('graceful-fs') +const JSONStream = require('JSONStream') +const log = require('npmlog') +const mkdir = BB.promisify(require('mkdirp')) +const ms = require('mississippi') +const npmFetch = require('libnpm/fetch') +const path = require('path') +const sortedUnionStream = require('sorted-union-stream') +const url = require('url') +const writeStreamAtomic = require('fs-write-stream-atomic') + +const statAsync = BB.promisify(fs.stat) + +const APMOpts = figgyPudding({ + cache: {}, + registry: {} +}) // Returns a sorted stream of all package metadata. Internally, takes care of // maintaining its metadata cache and making partial or full remote requests, // according to staleness, validity, etc. @@ -27,63 +34,70 @@ var gunzip = require('../utils/gunzip-maybe') // 4. It must include all entries that exist in the metadata endpoint as of // the value in `_updated` module.exports = allPackageMetadata -function allPackageMetadata (staleness) { - var stream = ms.through.obj() - - mapToRegistry('-/all', npm.config, function (er, uri, auth) { - if (er) return stream.emit('error', er) - - var cacheBase = cacheFile(npm.config.get('cache'))(uri) - var cachePath = path.join(cacheBase, '.cache.json') +function allPackageMetadata (opts) { + const staleness = opts.staleness + const stream = ms.through.obj() - createEntryStream(cachePath, uri, auth, staleness, function (err, entryStream, latest, newEntries) { - if (err) return stream.emit('error', err) - log.silly('all-package-metadata', 'entry stream created') - if (entryStream && newEntries) { - createCacheWriteStream(cachePath, latest, function (err, writeStream) { - if (err) return stream.emit('error', err) - log.silly('all-package-metadata', 'output stream created') - ms.pipeline.obj(entryStream, writeStream, stream) - }) - } else if (entryStream) { - ms.pipeline.obj(entryStream, stream) - } else { - stream.emit('error', new Error('No search sources available')) - } - }) - }) + opts = APMOpts(opts) + const cacheBase = cacheFile(path.resolve(path.dirname(opts.cache)))(url.resolve(opts.registry, '/-/all')) + const cachePath = path.join(cacheBase, '.cache.json') + createEntryStream( + cachePath, staleness, opts + ).then(({entryStream, latest, newEntries}) => { + log.silly('all-package-metadata', 'entry stream created') + if (entryStream && newEntries) { + return createCacheWriteStream(cachePath, latest, opts).then(writer => { + log.silly('all-package-metadata', 'output stream created') + ms.pipeline.obj(entryStream, writer, stream) + }) + } else if (entryStream) { + ms.pipeline.obj(entryStream, stream) + } else { + stream.emit('error', new Error('No search sources available')) + } + }).catch(err => stream.emit('error', err)) return stream } // Creates a stream of the latest available package metadata. // Metadata will come from a combination of the local cache and remote data. module.exports._createEntryStream = createEntryStream -function createEntryStream (cachePath, uri, auth, staleness, cb) { - createCacheEntryStream(cachePath, function (err, cacheStream, cacheLatest) { +function createEntryStream (cachePath, staleness, opts) { + return createCacheEntryStream( + cachePath, opts + ).catch(err => { + log.warn('', 'Failed to read search cache. Rebuilding') + log.silly('all-package-metadata', 'cache read error: ', err) + return {} + }).then(({ + updateStream: cacheStream, + updatedLatest: cacheLatest + }) => { cacheLatest = cacheLatest || 0 - if (err) { - log.warn('', 'Failed to read search cache. Rebuilding') - log.silly('all-package-metadata', 'cache read error: ', err) - } - createEntryUpdateStream(uri, auth, staleness, cacheLatest, function (err, updateStream, updatedLatest) { + return createEntryUpdateStream(staleness, cacheLatest, opts).catch(err => { + log.warn('', 'Search data request failed, search might be stale') + log.silly('all-package-metadata', 'update request error: ', err) + return {} + }).then(({updateStream, updatedLatest}) => { updatedLatest = updatedLatest || 0 - var latest = updatedLatest || cacheLatest + const latest = updatedLatest || cacheLatest if (!cacheStream && !updateStream) { - return cb(new Error('No search sources available')) - } - if (err) { - log.warn('', 'Search data request failed, search might be stale') - log.silly('all-package-metadata', 'update request error: ', err) + throw new Error('No search sources available') } if (cacheStream && updateStream) { // Deduped, unioned, sorted stream from the combination of both. - cb(null, - createMergedStream(cacheStream, updateStream), + return { + entryStream: createMergedStream(cacheStream, updateStream), latest, - !!updatedLatest) + newEntries: !!updatedLatest + } } else { // Either one works if one or the other failed - cb(null, cacheStream || updateStream, latest, !!updatedLatest) + return { + entryStream: cacheStream || updateStream, + latest, + newEntries: !!updatedLatest + } } }) }) @@ -96,66 +110,51 @@ function createEntryStream (cachePath, uri, auth, staleness, cb) { module.exports._createMergedStream = createMergedStream function createMergedStream (a, b) { linkStreams(a, b) - return sortedUnionStream(b, a, function (pkg) { return pkg.name }) + return sortedUnionStream(b, a, ({name}) => name) } // Reads the local index and returns a stream that spits out package data. module.exports._createCacheEntryStream = createCacheEntryStream -function createCacheEntryStream (cacheFile, cb) { +function createCacheEntryStream (cacheFile, opts) { log.verbose('all-package-metadata', 'creating entry stream from local cache') log.verbose('all-package-metadata', cacheFile) - fs.stat(cacheFile, function (err, stat) { - if (err) return cb(err) + return statAsync(cacheFile).then(stat => { // TODO - This isn't very helpful if `cacheFile` is empty or just `{}` - var entryStream = ms.pipeline.obj( + const entryStream = ms.pipeline.obj( fs.createReadStream(cacheFile), - jsonstream.parse('*'), + JSONStream.parse('*'), // I believe this passthrough is necessary cause `jsonstream` returns // weird custom streams that behave funny sometimes. ms.through.obj() ) - extractUpdated(entryStream, 'cached-entry-stream', cb) + return extractUpdated(entryStream, 'cached-entry-stream', opts) }) } // Stream of entry updates from the server. If `latest` is `0`, streams the // entire metadata object from the registry. module.exports._createEntryUpdateStream = createEntryUpdateStream -function createEntryUpdateStream (all, auth, staleness, latest, cb) { +function createEntryUpdateStream (staleness, latest, opts) { log.verbose('all-package-metadata', 'creating remote entry stream') - var params = { - timeout: 600, - follow: true, - staleOk: true, - auth: auth, - streaming: true - } - var partialUpdate = false + let partialUpdate = false + let uri = '/-/all' if (latest && (Date.now() - latest < (staleness * 1000))) { // Skip the request altogether if our `latest` isn't stale. log.verbose('all-package-metadata', 'Local data up to date, skipping update') - return cb(null) + return BB.resolve({}) } else if (latest === 0) { log.warn('', 'Building the local index for the first time, please be patient') log.verbose('all-package-metadata', 'No cached data: requesting full metadata db') } else { log.verbose('all-package-metadata', 'Cached data present with timestamp:', latest, 'requesting partial index update') - all += '/since?stale=update_after&startkey=' + latest + uri += '/since?stale=update_after&startkey=' + latest partialUpdate = true } - npm.registry.request(all, params, function (er, res) { - if (er) return cb(er) + return npmFetch(uri, opts).then(res => { log.silly('all-package-metadata', 'request stream opened, code:', res.statusCode) - // NOTE - The stream returned by `request` seems to be very persnickety - // and this is almost a magic incantation to get it to work. - // Modify how `res` is used here at your own risk. - var entryStream = ms.pipeline.obj( - res, - ms.through(function (chunk, enc, cb) { - cb(null, chunk) - }), - gunzip(), - jsonstream.parse('*', function (pkg, key) { + let entryStream = ms.pipeline.obj( + res.body, + JSONStream.parse('*', (pkg, key) => { if (key[0] === '_updated' || key[0][0] !== '_') { return pkg } @@ -164,9 +163,12 @@ function createEntryUpdateStream (all, auth, staleness, latest, cb) { if (partialUpdate) { // The `/all/since` endpoint doesn't return `_updated`, so we // just use the request's own timestamp. - cb(null, entryStream, Date.parse(res.headers.date)) + return { + updateStream: entryStream, + updatedLatest: Date.parse(res.headers.get('date')) + } } else { - extractUpdated(entryStream, 'entry-update-stream', cb) + return extractUpdated(entryStream, 'entry-update-stream', opts) } }) } @@ -175,36 +177,37 @@ function createEntryUpdateStream (all, auth, staleness, latest, cb) { // first returned entries. This is the "latest" unix timestamp for the metadata // in question. This code does a bit of juggling with the data streams // so that we can pretend that field doesn't exist, but still extract `latest` -function extractUpdated (entryStream, label, cb) { - cb = once(cb) +function extractUpdated (entryStream, label, opts) { log.silly('all-package-metadata', 'extracting latest') - function nope (msg) { - return function () { - log.warn('all-package-metadata', label, msg) - entryStream.removeAllListeners() - entryStream.destroy() - cb(new Error(msg)) - } - } - var onErr = nope('Failed to read stream') - var onEnd = nope('Empty or invalid stream') - entryStream.on('error', onErr) - entryStream.on('end', onEnd) - entryStream.once('data', function (latest) { - log.silly('all-package-metadata', 'got first stream entry for', label, latest) - entryStream.removeListener('error', onErr) - entryStream.removeListener('end', onEnd) - // Because `.once()` unpauses the stream, we re-pause it after the first - // entry so we don't vomit entries into the void. - entryStream.pause() - if (typeof latest === 'number') { - // The extra pipeline is to return a stream that will implicitly unpause - // after having an `.on('data')` listener attached, since using this - // `data` event broke its initial state. - cb(null, ms.pipeline.obj(entryStream, ms.through.obj()), latest) - } else { - cb(new Error('expected first entry to be _updated')) + return new BB((resolve, reject) => { + function nope (msg) { + return function () { + log.warn('all-package-metadata', label, msg) + entryStream.removeAllListeners() + entryStream.destroy() + reject(new Error(msg)) + } } + const onErr = nope('Failed to read stream') + const onEnd = nope('Empty or invalid stream') + entryStream.on('error', onErr) + entryStream.on('end', onEnd) + entryStream.once('data', latest => { + log.silly('all-package-metadata', 'got first stream entry for', label, latest) + entryStream.removeListener('error', onErr) + entryStream.removeListener('end', onEnd) + if (typeof latest === 'number') { + // The extra pipeline is to return a stream that will implicitly unpause + // after having an `.on('data')` listener attached, since using this + // `data` event broke its initial state. + resolve({ + updateStream: entryStream.pipe(ms.through.obj()), + updatedLatest: latest + }) + } else { + reject(new Error('expected first entry to be _updated')) + } + }) }) } @@ -213,44 +216,43 @@ function extractUpdated (entryStream, label, cb) { // The stream is also passthrough, so entries going through it will also // be output from it. module.exports._createCacheWriteStream = createCacheWriteStream -function createCacheWriteStream (cacheFile, latest, cb) { - _ensureCacheDirExists(cacheFile, function (err) { - if (err) return cb(err) +function createCacheWriteStream (cacheFile, latest, opts) { + return _ensureCacheDirExists(cacheFile, opts).then(() => { log.silly('all-package-metadata', 'creating output stream') - var outStream = _createCacheOutStream() - var cacheFileStream = writeStreamAtomic(cacheFile) - var inputStream = _createCacheInStream(cacheFileStream, outStream, latest) + const outStream = _createCacheOutStream() + const cacheFileStream = writeStreamAtomic(cacheFile) + const inputStream = _createCacheInStream( + cacheFileStream, outStream, latest + ) // Glue together the various streams so they fail together. // `cacheFileStream` errors are already handled by the `inputStream` // pipeline - var errEmitted = false - linkStreams(inputStream, outStream, function () { errEmitted = true }) + let errEmitted = false + linkStreams(inputStream, outStream, () => { errEmitted = true }) - cacheFileStream.on('close', function () { !errEmitted && outStream.end() }) + cacheFileStream.on('close', () => !errEmitted && outStream.end()) - cb(null, ms.duplex.obj(inputStream, outStream)) + return ms.duplex.obj(inputStream, outStream) }) } -function _ensureCacheDirExists (cacheFile, cb) { +function _ensureCacheDirExists (cacheFile, opts) { var cacheBase = path.dirname(cacheFile) log.silly('all-package-metadata', 'making sure cache dir exists at', cacheBase) - correctMkdir(npm.cache, function (er, st) { - if (er) return cb(er) - mkdir(cacheBase, function (er, made) { - if (er) return cb(er) - chownr(made || cacheBase, st.uid, st.gid, cb) + return correctMkdir(opts.cache).then(st => { + return mkdir(cacheBase).then(made => { + return chownr(made || cacheBase, st.uid, st.gid) }) }) } function _createCacheOutStream () { + // NOTE: this looks goofy, but it's necessary in order to get + // JSONStream to play nice with the rest of everything. return ms.pipeline.obj( - // These two passthrough `through` streams compensate for some - // odd behavior with `jsonstream`. ms.through(), - jsonstream.parse('*', function (obj, key) { + JSONStream.parse('*', (obj, key) => { // This stream happens to get _updated passed through it, for // implementation reasons. We make sure to filter it out cause // the fact that it comes t @@ -263,9 +265,9 @@ function _createCacheOutStream () { } function _createCacheInStream (writer, outStream, latest) { - var updatedWritten = false - var inStream = ms.pipeline.obj( - ms.through.obj(function (pkg, enc, cb) { + let updatedWritten = false + const inStream = ms.pipeline.obj( + ms.through.obj((pkg, enc, cb) => { if (!updatedWritten && typeof pkg === 'number') { // This is the `_updated` value getting sent through. updatedWritten = true @@ -277,13 +279,11 @@ function _createCacheInStream (writer, outStream, latest) { cb(null, [pkg.name, pkg]) } }), - jsonstream.stringifyObject('{', ',', '}'), - ms.through(function (chunk, enc, cb) { + JSONStream.stringifyObject('{', ',', '}'), + ms.through((chunk, enc, cb) => { // This tees off the buffer data to `outStream`, and then continues // the pipeline as usual - outStream.write(chunk, enc, function () { - cb(null, chunk) - }) + outStream.write(chunk, enc, () => cb(null, chunk)) }), // And finally, we write to the cache file. writer @@ -300,14 +300,14 @@ function linkStreams (a, b, cb) { if (err !== lastError) { lastError = err b.emit('error', err) - cb(err) + cb && cb(err) } }) b.on('error', function (err) { if (err !== lastError) { lastError = err a.emit('error', err) - cb(err) + cb && cb(err) } }) } diff --git a/lib/search/all-package-search.js b/lib/search/all-package-search.js index 7a893d517b82c..fef343bcbc3ba 100644 --- a/lib/search/all-package-search.js +++ b/lib/search/all-package-search.js @@ -8,7 +8,7 @@ function allPackageSearch (opts) { // Get a stream with *all* the packages. This takes care of dealing // with the local cache as well, but that's an internal detail. - var allEntriesStream = allPackageMetadata(opts.staleness) + var allEntriesStream = allPackageMetadata(opts) // Grab a stream that filters those packages according to given params. var filterStream = streamFilter(function (pkg) { diff --git a/lib/search/esearch.js b/lib/search/esearch.js deleted file mode 100644 index f4beb7ade66b1..0000000000000 --- a/lib/search/esearch.js +++ /dev/null @@ -1,64 +0,0 @@ -'use strict' - -var npm = require('../npm.js') -var log = require('npmlog') -var mapToRegistry = require('../utils/map-to-registry.js') -var jsonstream = require('JSONStream') -var ms = require('mississippi') -var gunzip = require('../utils/gunzip-maybe') - -module.exports = esearch - -function esearch (opts) { - var stream = ms.through.obj() - - mapToRegistry('-/v1/search', npm.config, function (er, uri, auth) { - if (er) return stream.emit('error', er) - createResultStream(uri, auth, opts, function (err, resultStream) { - if (err) return stream.emit('error', err) - ms.pipeline.obj(resultStream, stream) - }) - }) - return stream -} - -function createResultStream (uri, auth, opts, cb) { - log.verbose('esearch', 'creating remote entry stream') - var params = { - timeout: 600, - follow: true, - staleOk: true, - auth: auth, - streaming: true - } - var q = buildQuery(opts) - npm.registry.request(uri + '?text=' + encodeURIComponent(q) + '&size=' + opts.limit, params, function (err, res) { - if (err) return cb(err) - log.silly('esearch', 'request stream opened, code:', res.statusCode) - // NOTE - The stream returned by `request` seems to be very persnickety - // and this is almost a magic incantation to get it to work. - // Modify how `res` is used here at your own risk. - var entryStream = ms.pipeline.obj( - res, - ms.through(function (chunk, enc, cb) { - cb(null, chunk) - }), - gunzip(), - jsonstream.parse('objects.*.package', function (data) { - return { - name: data.name, - description: data.description, - maintainers: data.maintainers, - keywords: data.keywords, - version: data.version, - date: data.date ? new Date(data.date) : null - } - }) - ) - return cb(null, entryStream) - }) -} - -function buildQuery (opts) { - return opts.include.join(' ') -} diff --git a/test/tap/all-package-metadata-cache-stream-unit.js b/test/tap/all-package-metadata-cache-stream-unit.js index 51be836769050..0b4dd0e26d5ce 100644 --- a/test/tap/all-package-metadata-cache-stream-unit.js +++ b/test/tap/all-package-metadata-cache-stream-unit.js @@ -1,18 +1,20 @@ 'use strict' require('../common-tap.js') -var test = require('tap').test -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var path = require('path') -var ms = require('mississippi') -var Tacks = require('tacks') -var File = Tacks.File -var _createCacheEntryStream = require('../../lib/search/all-package-metadata.js')._createCacheEntryStream +const getStream = require('get-stream') +const mkdirp = require('mkdirp') +const path = require('path') +const rimraf = require('rimraf') +const Tacks = require('tacks') +const {test} = require('tap') -var PKG_DIR = path.resolve(__dirname, 'create-cache-entry-stream') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') +const {File} = Tacks + +const _createCacheEntryStream = require('../../lib/search/all-package-metadata.js')._createCacheEntryStream + +const PKG_DIR = path.resolve(__dirname, 'create-cache-entry-stream') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache') function setup () { mkdirp.sync(CACHE_DIR) @@ -22,10 +24,10 @@ function cleanup () { rimraf.sync(PKG_DIR) } -test('createCacheEntryStream basic', function (t) { +test('createCacheEntryStream basic', t => { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File({ + const cachePath = path.join(CACHE_DIR, '.cache.json') + const fixture = new Tacks(File({ '_updated': 1234, bar: { name: 'bar', @@ -37,16 +39,13 @@ test('createCacheEntryStream basic', function (t) { } })) fixture.create(cachePath) - _createCacheEntryStream(cachePath, function (err, stream, latest) { - if (err) throw err + return _createCacheEntryStream(cachePath, {}).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.equals(latest, 1234, '`latest` correctly extracted') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err + return getStream.array(stream).then(results => { t.deepEquals(results, [{ name: 'bar', version: '1.0.0' @@ -55,82 +54,54 @@ test('createCacheEntryStream basic', function (t) { version: '1.0.0' }]) cleanup() - t.done() }) }) }) -test('createCacheEntryStream empty cache', function (t) { +test('createCacheEntryStream empty cache', t => { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File({})) + const cachePath = path.join(CACHE_DIR, '.cache.json') + const fixture = new Tacks(File({})) fixture.create(cachePath) - _createCacheEntryStream(cachePath, function (err, stream, latest) { - t.ok(err, 'returned an error because there was no _updated') - t.match(err.message, /Empty or invalid stream/, 'useful error message') - t.notOk(stream, 'no stream returned') - t.notOk(latest, 'no latest returned') - cleanup() - t.done() - }) + return _createCacheEntryStream(cachePath, {}).then( + () => { throw new Error('should not succeed') }, + err => { + t.ok(err, 'returned an error because there was no _updated') + t.match(err.message, /Empty or invalid stream/, 'useful error message') + cleanup() + } + ) }) -test('createCacheEntryStream no entry cache', function (t) { +test('createCacheEntryStream no entry cache', t => { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File({ + const cachePath = path.join(CACHE_DIR, '.cache.json') + const fixture = new Tacks(File({ '_updated': 1234 })) fixture.create(cachePath) - _createCacheEntryStream(cachePath, function (err, stream, latest) { - if (err) throw err + return _createCacheEntryStream(cachePath, {}).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.equals(latest, 1234, '`latest` correctly extracted') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err + return getStream.array(stream).then(results => { t.deepEquals(results, [], 'no results') cleanup() - t.done() }) }) }) -test('createCacheEntryStream missing cache', function (t) { +test('createCacheEntryStream missing cache', t => { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - _createCacheEntryStream(cachePath, function (err, stream, latest) { - t.ok(err, 'returned an error because there was no cache') - t.equals(err.code, 'ENOENT', 'useful error message') - t.notOk(stream, 'no stream returned') - t.notOk(latest, 'no latest returned') - cleanup() - t.done() - }) -}) - -test('createCacheEntryStream bad syntax', function (t) { - setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File('{"_updated": 1234, uh oh')) - fixture.create(cachePath) - _createCacheEntryStream(cachePath, function (err, stream, latest) { - if (err) throw err - t.equals(latest, 1234, '`latest` correctly extracted') - t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - t.ok(err, 'stream errored') - t.match(err.message, /Invalid JSON/i, 'explains there\'s a syntax error') - t.deepEquals(results, [], 'no results') + const cachePath = path.join(CACHE_DIR, '.cache.json') + return _createCacheEntryStream(cachePath, {}).then( + () => { throw new Error('should not succeed') }, + err => { + t.ok(err, 'returned an error because there was no cache') + t.equals(err.code, 'ENOENT', 'useful error message') cleanup() - t.done() - }) - }) + } + ) }) diff --git a/test/tap/all-package-metadata-entry-stream-unit.js b/test/tap/all-package-metadata-entry-stream-unit.js index 0e02f848246ce..4e916229cd852 100644 --- a/test/tap/all-package-metadata-entry-stream-unit.js +++ b/test/tap/all-package-metadata-entry-stream-unit.js @@ -1,23 +1,23 @@ 'use strict' -var common = require('../common-tap.js') -var npm = require('../../') -var test = require('tap').test -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var path = require('path') -var mr = require('npm-registry-mock') -var ms = require('mississippi') -var Tacks = require('tacks') -var File = Tacks.File +const common = require('../common-tap.js') +const getStream = require('get-stream') +const mkdirp = require('mkdirp') +const mr = require('npm-registry-mock') +const npm = require('../../') +const path = require('path') +const rimraf = require('rimraf') +const Tacks = require('tacks') +const test = require('tap').test -var _createEntryStream = require('../../lib/search/all-package-metadata.js')._createEntryStream +const {File} = Tacks -var ALL = common.registry + '/-/all' -var PKG_DIR = path.resolve(__dirname, 'create-entry-update-stream') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') +const _createEntryStream = require('../../lib/search/all-package-metadata.js')._createEntryStream -var server +const PKG_DIR = path.resolve(__dirname, 'create-entry-update-stream') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache') + +let server function setup () { mkdirp.sync(CACHE_DIR) @@ -27,10 +27,11 @@ function cleanup () { rimraf.sync(PKG_DIR) } -test('setup', function (t) { - mr({port: common.port, throwOnUnmatched: true}, function (err, s) { +test('setup', t => { + cleanup() + mr({port: common.port, throwOnUnmatched: true}, (err, s) => { t.ifError(err, 'registry mocked successfully') - npm.load({ cache: CACHE_DIR, registry: common.registry }, function (err) { + npm.load({ cache: CACHE_DIR, registry: common.registry }, err => { t.ifError(err, 'npm loaded successfully') server = s t.pass('all set up') @@ -39,10 +40,10 @@ test('setup', function (t) { }) }) -test('createEntryStream full request', function (t) { +test('createEntryStream full request', t => { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') - var dataTime = +(new Date()) + const cachePath = path.join(CACHE_DIR, '.cache.json') + const dataTime = +(new Date()) server.get('/-/all').once().reply(200, { '_updated': dataTime, 'bar': { name: 'bar', version: '1.0.0' }, @@ -50,37 +51,36 @@ test('createEntryStream full request', function (t) { }, { date: 1234 // should never be used. }) - _createEntryStream(cachePath, ALL, {}, 600, function (err, stream, latest, newEntries) { - if (err) throw err + return _createEntryStream(cachePath, 600, { + registry: common.registry + }).then(({ + entryStream: stream, + latest, + newEntries + }) => { t.equals(latest, dataTime, '`latest` correctly extracted') t.ok(newEntries, 'new entries need to be written to cache') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err - t.deepEquals(results, [{ - name: 'bar', - version: '1.0.0' - }, { - name: 'foo', - version: '1.0.0' - }]) - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals(results, [{ + name: 'bar', + version: '1.0.0' + }, { + name: 'foo', + version: '1.0.0' + }]) + server.done() + cleanup() }) }) test('createEntryStream cache only', function (t) { setup() - var now = Date.now() - var cacheTime = now - 100000 - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File({ + const now = Date.now() + const cacheTime = now - 100000 + const cachePath = path.join(CACHE_DIR, '.cache.json') + const fixture = new Tacks(File({ '_updated': cacheTime, bar: { name: 'bar', version: '1.0.0' }, cool: { name: 'cool', version: '1.0.0' }, @@ -88,33 +88,32 @@ test('createEntryStream cache only', function (t) { other: { name: 'other', version: '1.0.0' } })) fixture.create(cachePath) - _createEntryStream(cachePath, ALL, {}, 600, function (err, stream, latest, newEntries) { - if (err) throw err + return _createEntryStream(cachePath, 600, { + registry: common.registry + }).then(({ + entryStream: stream, + latest, + newEntries + }) => { t.equals(latest, cacheTime, '`latest` is cache time') t.ok(stream, 'returned a stream') t.notOk(newEntries, 'cache only means no need to write to cache') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - t.ifError(err, 'stream finished without error') - t.deepEquals( - results.map(function (pkg) { return pkg.name }), - ['bar', 'cool', 'foo', 'other'], - 'packages deduped and sorted' - ) - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals( + results.map(function (pkg) { return pkg.name }), + ['bar', 'cool', 'foo', 'other'], + 'packages deduped and sorted' + ) + server.done() + cleanup() }) }) test('createEntryStream merged stream', function (t) { setup() - var now = Date.now() - var cacheTime = now - 6000000 + const now = Date.now() + const cacheTime = now - 6000000 server.get('/-/all/since?stale=update_after&startkey=' + cacheTime).once().reply(200, { 'bar': { name: 'bar', version: '2.0.0' }, 'car': { name: 'car', version: '1.0.0' }, @@ -122,8 +121,8 @@ test('createEntryStream merged stream', function (t) { }, { date: (new Date(now)).toISOString() }) - var cachePath = path.join(CACHE_DIR, '.cache.json') - var fixture = new Tacks(File({ + const cachePath = path.join(CACHE_DIR, '.cache.json') + const fixture = new Tacks(File({ '_updated': cacheTime, bar: { name: 'bar', version: '1.0.0' }, cool: { name: 'cool', version: '1.0.0' }, @@ -131,50 +130,54 @@ test('createEntryStream merged stream', function (t) { other: { name: 'other', version: '1.0.0' } })) fixture.create(cachePath) - _createEntryStream(cachePath, ALL, {}, 600, function (err, stream, latest, newEntries) { - if (err) throw err + return _createEntryStream(cachePath, 600, { + registry: common.registry + }).then(({ + entryStream: stream, + latest, + newEntries + }) => { t.equals(latest, now, '`latest` correctly extracted from header') t.ok(stream, 'returned a stream') t.ok(newEntries, 'cache update means entries should be written') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - t.ifError(err, 'stream finished without error') - t.deepEquals( - results.map(function (pkg) { return pkg.name }), - ['bar', 'car', 'cool', 'foo', 'other'], - 'packages deduped and sorted' - ) - t.deepEquals(results[0], { - name: 'bar', - version: '2.0.0' - }, 'update stream version wins on dedupe') - t.deepEquals(results[3], { - name: 'foo', - version: '1.0.0' - }, 'update stream version wins on dedupe even when the newer one has a lower semver.') - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals( + results.map(function (pkg) { return pkg.name }), + ['bar', 'car', 'cool', 'foo', 'other'], + 'packages deduped and sorted' + ) + t.deepEquals(results[0], { + name: 'bar', + version: '2.0.0' + }, 'update stream version wins on dedupe') + t.deepEquals(results[3], { + name: 'foo', + version: '1.0.0' + }, 'update stream version wins on dedupe even when the newer one has a lower semver.') + server.done() + cleanup() }) }) test('createEntryStream no sources', function (t) { setup() - var cachePath = path.join(CACHE_DIR, '.cache.json') + const cachePath = path.join(CACHE_DIR, '.cache.json') server.get('/-/all').once().reply(404, {}) - _createEntryStream(cachePath, ALL, {}, 600, function (err, stream, latest, newEntries) { + return _createEntryStream(cachePath, 600, { + registry: common.registry + }).then(({ + entryStream: stream, + latest, + newEntries + }) => { + throw new Error('should not succeed') + }, err => { t.ok(err, 'no sources, got an error') - t.notOk(stream, 'no stream returned') - t.notOk(latest, 'no latest returned') - t.notOk(newEntries, 'no entries need to be written') t.match(err.message, /No search sources available/, 'useful error message') + }).then(() => { server.done() cleanup() - t.end() }) }) diff --git a/test/tap/all-package-metadata-update-stream-unit.js b/test/tap/all-package-metadata-update-stream-unit.js index b9cf337eb9a08..2c08ac347ed69 100644 --- a/test/tap/all-package-metadata-update-stream-unit.js +++ b/test/tap/all-package-metadata-update-stream-unit.js @@ -1,17 +1,16 @@ 'use strict' -var common = require('../common-tap.js') -var npm = require('../../') -var test = require('tap').test -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var path = require('path') -var mr = require('npm-registry-mock') -var ms = require('mississippi') +const common = require('../common-tap.js') +const getStream = require('get-stream') +const npm = require('../../') +const test = require('tap').test +const mkdirp = require('mkdirp') +const rimraf = require('rimraf') +const path = require('path') +const mr = require('npm-registry-mock') var _createEntryUpdateStream = require('../../lib/search/all-package-metadata.js')._createEntryUpdateStream -var ALL = common.registry + '/-/all' var PKG_DIR = path.resolve(__dirname, 'create-entry-update-stream') var CACHE_DIR = path.resolve(PKG_DIR, 'cache') @@ -46,27 +45,25 @@ test('createEntryUpdateStream full request', function (t) { }, { date: Date.now() // should never be used. }) - _createEntryUpdateStream(ALL, {}, 600, 0, function (err, stream, latest) { - if (err) throw err + return _createEntryUpdateStream(600, 0, { + registry: common.registry + }).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.equals(latest, 1234, '`latest` correctly extracted') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err - t.deepEquals(results, [{ - name: 'bar', - version: '1.0.0' - }, { - name: 'foo', - version: '1.0.0' - }]) - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals(results, [{ + name: 'bar', + version: '1.0.0' + }, { + name: 'foo', + version: '1.0.0' + }]) + server.done() + cleanup() }) }) @@ -79,27 +76,25 @@ test('createEntryUpdateStream partial update', function (t) { }, { date: (new Date(now)).toISOString() }) - _createEntryUpdateStream(ALL, {}, 600, 1234, function (err, stream, latest) { - if (err) throw err + return _createEntryUpdateStream(600, 1234, { + registry: common.registry + }).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.equals(latest, now, '`latest` correctly extracted from header') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err - t.deepEquals(results, [{ - name: 'bar', - version: '1.0.0' - }, { - name: 'foo', - version: '1.0.0' - }]) - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals(results, [{ + name: 'bar', + version: '1.0.0' + }, { + name: 'foo', + version: '1.0.0' + }]) + server.done() + cleanup() }) }) @@ -113,27 +108,26 @@ test('createEntryUpdateStream authed request', function (t) { }, { date: Date.now() // should never be used. }) - _createEntryUpdateStream(ALL, { token: token }, 600, 0, function (err, stream, latest) { - if (err) throw err + return _createEntryUpdateStream(600, 0, { + registry: common.registry, + token + }).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.equals(latest, 1234, '`latest` correctly extracted') t.ok(stream, 'returned a stream') - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err - t.deepEquals(results, [{ - name: 'bar', - version: '1.0.0' - }, { - name: 'foo', - version: '1.0.0' - }]) - server.done() - cleanup() - t.end() - }) + return getStream.array(stream) + }).then(results => { + t.deepEquals(results, [{ + name: 'bar', + version: '1.0.0' + }, { + name: 'foo', + version: '1.0.0' + }]) + server.done() + cleanup() }) }) @@ -143,14 +137,17 @@ test('createEntryUpdateStream bad auth', function (t) { server.get('/-/all', { authorization: 'Bearer ' + token }).once().reply(401, { error: 'unauthorized search request' }) - _createEntryUpdateStream(ALL, { token: token }, 600, 0, function (err, stream, latest) { + return _createEntryUpdateStream(600, 0, { + registry: common.registry, + token + }).then(() => { + throw new Error('should not succeed') + }, err => { t.ok(err, 'got an error from auth failure') - t.notOk(stream, 'no stream returned') - t.notOk(latest, 'no latest returned') t.match(err, /unauthorized/, 'failure message from request used') + }).then(() => { server.done() cleanup() - t.end() }) }) @@ -158,8 +155,12 @@ test('createEntryUpdateStream not stale', function (t) { setup() var now = Date.now() var staleness = 600 - _createEntryUpdateStream(ALL, {}, staleness, now, function (err, stream, latest) { - t.ifError(err, 'completed successfully') + return _createEntryUpdateStream(staleness, now, { + registry: common.registry + }).then(({ + updateStream: stream, + updatedLatest: latest + }) => { t.notOk(stream, 'no stream returned') t.notOk(latest, 'no latest returned') server.done() diff --git a/test/tap/all-package-metadata-write-stream-unit.js b/test/tap/all-package-metadata-write-stream-unit.js index 410f7f9e9d927..94bb7413f1b32 100644 --- a/test/tap/all-package-metadata-write-stream-unit.js +++ b/test/tap/all-package-metadata-write-stream-unit.js @@ -1,18 +1,19 @@ 'use strict' -var common = require('../common-tap.js') -var npm = require('../../') -var test = require('tap').test -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var path = require('path') -var fs = require('fs') -var ms = require('mississippi') +const common = require('../common-tap.js') +const getStream = require('get-stream') +const npm = require('../../') +const test = require('tap').test +const mkdirp = require('mkdirp') +const rimraf = require('rimraf') +const path = require('path') +const fs = require('fs') +const ms = require('mississippi') -var _createCacheWriteStream = require('../../lib/search/all-package-metadata.js')._createCacheWriteStream +const _createCacheWriteStream = require('../../lib/search/all-package-metadata.js')._createCacheWriteStream -var PKG_DIR = path.resolve(__dirname, 'create-cache-write-stream') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') +const PKG_DIR = path.resolve(__dirname, 'create-cache-write-stream') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache') function setup () { mkdirp.sync(CACHE_DIR) @@ -46,60 +47,54 @@ test('createCacheEntryStream basic', function (t) { { name: 'foo', version: '1.0.0' } ] var srcStream = fromArray(src) - _createCacheWriteStream(cachePath, latest, function (err, stream) { - if (err) throw err + return _createCacheWriteStream(cachePath, latest, { + cache: CACHE_DIR + }).then(stream => { t.ok(stream, 'returned a stream') stream = ms.pipeline.obj(srcStream, stream) - var results = [] - stream.on('data', function (pkg) { - results.push(pkg) - }) - ms.finished(stream, function (err) { - if (err) throw err - t.deepEquals(results, [{ + return getStream.array(stream) + }).then(results => { + t.deepEquals(results, [{ + name: 'bar', + version: '1.0.0' + }, { + name: 'foo', + version: '1.0.0' + }]) + var fileData = JSON.parse(fs.readFileSync(cachePath)) + t.ok(fileData, 'cache contents written to the right file') + t.deepEquals(fileData, { + '_updated': latest, + bar: { name: 'bar', version: '1.0.0' - }, { + }, + foo: { name: 'foo', version: '1.0.0' - }]) - var fileData = JSON.parse(fs.readFileSync(cachePath)) - t.ok(fileData, 'cache contents written to the right file') - t.deepEquals(fileData, { - '_updated': latest, - bar: { - name: 'bar', - version: '1.0.0' - }, - foo: { - name: 'foo', - version: '1.0.0' - } - }, 'cache contents based on what was written') - cleanup() - t.done() - }) + } + }, 'cache contents based on what was written') + cleanup() }) }) test('createCacheEntryStream no entries', function (t) { - cleanup() // wipe out the cache dir - var cachePath = path.join(CACHE_DIR, '.cache.json') + setup() + const cachePath = path.join(CACHE_DIR, '.cache.json') var latest = 12345 - var src = [] - var srcStream = fromArray(src) - _createCacheWriteStream(cachePath, latest, function (err, stream) { - if (err) throw err + const src = [] + const srcStream = fromArray(src) + return _createCacheWriteStream(cachePath, latest, { + cache: CACHE_DIR + }).then(stream => { t.ok(stream, 'returned a stream') stream = ms.pipeline.obj(srcStream, stream) stream.resume() - ms.finished(stream, function (err) { - if (err) throw err - var fileData = JSON.parse(fs.readFileSync(cachePath)) - t.ok(fileData, 'cache file exists and has stuff in it') - cleanup() - t.done() - }) + return getStream(stream) + }).then(() => { + const fileData = JSON.parse(fs.readFileSync(cachePath)) + t.ok(fileData, 'cache file exists and has stuff in it') + cleanup() }) }) @@ -109,22 +104,19 @@ test('createCacheEntryStream missing cache dir', function (t) { var latest = 12345 var src = [] var srcStream = fromArray(src) - _createCacheWriteStream(cachePath, latest, function (err, stream) { - if (err) throw err + return _createCacheWriteStream(cachePath, latest, { + cache: CACHE_DIR + }).then(stream => { t.ok(stream, 'returned a stream') stream = ms.pipeline.obj(srcStream, stream) - stream.on('data', function (pkg) { - t.notOk(pkg, 'stream should not have output any data') - }) - ms.finished(stream, function (err) { - if (err) throw err - var fileData = JSON.parse(fs.readFileSync(cachePath)) - t.ok(fileData, 'cache contents written to the right file') - t.deepEquals(fileData, { - '_updated': latest - }, 'cache still contains `_updated`') - cleanup() - t.done() - }) + return getStream.array(stream) + }).then(res => { + t.deepEqual(res, [], 'no data returned') + var fileData = JSON.parse(fs.readFileSync(cachePath)) + t.ok(fileData, 'cache contents written to the right file') + t.deepEquals(fileData, { + '_updated': latest + }, 'cache still contains `_updated`') + cleanup() }) }) diff --git a/test/tap/all-package-metadata.js b/test/tap/all-package-metadata.js index 9b60822e4eb00..99d3fa26c52b8 100644 --- a/test/tap/all-package-metadata.js +++ b/test/tap/all-package-metadata.js @@ -1,26 +1,26 @@ 'use strict' -var common = require('../common-tap.js') -var npm = require('../../') -var test = require('tap').test -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var path = require('path') -var fs = require('fs') -var cacheFile = require('npm-cache-filename') -var mr = require('npm-registry-mock') -var ms = require('mississippi') -var Tacks = require('tacks') -var File = Tacks.File +const common = require('../common-tap.js') +const npm = require('../../') +const test = require('tap').test +const mkdirp = require('mkdirp') +const rimraf = require('rimraf') +const path = require('path') +const fs = require('fs') +const cacheFile = require('npm-cache-filename') +const mr = require('npm-registry-mock') +const ms = require('mississippi') +const Tacks = require('tacks') +const File = Tacks.File -var allPackageMetadata = require('../../lib/search/all-package-metadata.js') +const allPackageMetadata = require('../../lib/search/all-package-metadata.js') -var PKG_DIR = path.resolve(__dirname, 'update-index') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') -var cacheBase -var cachePath +const PKG_DIR = path.resolve(__dirname, path.basename(__filename, '.js'), 'update-index') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache', '_cacache') +let cacheBase +let cachePath -var server +let server function setup () { mkdirp.sync(cacheBase) @@ -33,9 +33,9 @@ function cleanup () { test('setup', function (t) { mr({port: common.port, throwOnUnmatched: true}, function (err, s) { t.ifError(err, 'registry mocked successfully') - npm.load({ cache: CACHE_DIR, registry: common.registry }, function (err) { + npm.load({ cache: path.dirname(CACHE_DIR), registry: common.registry }, function (err) { t.ifError(err, 'npm loaded successfully') - npm.config.set('cache', CACHE_DIR) + npm.config.set('cache', path.dirname(CACHE_DIR)) cacheBase = cacheFile(npm.config.get('cache'))(common.registry + '/-/all') cachePath = path.join(cacheBase, '.cache.json') server = s @@ -55,7 +55,11 @@ test('allPackageMetadata full request', function (t) { }, { date: updated }) - var stream = allPackageMetadata(600) + var stream = allPackageMetadata({ + cache: CACHE_DIR, + registry: common.registry, + staleness: 600 + }) t.ok(stream, 'returned a stream') var results = [] stream.on('data', function (pkg) { @@ -101,7 +105,11 @@ test('allPackageMetadata cache only', function (t) { } var fixture = new Tacks(File(cacheContents)) fixture.create(cachePath) - var stream = allPackageMetadata(10000000) + var stream = allPackageMetadata({ + cache: CACHE_DIR, + registry: common.registry, + staleness: 10000000 + }) t.ok(stream, 'returned a stream') var results = [] stream.on('data', function (pkg) { @@ -143,7 +151,11 @@ test('createEntryStream merged stream', function (t) { other: { name: 'other', version: '1.0.0' } })) fixture.create(cachePath) - var stream = allPackageMetadata(600) + var stream = allPackageMetadata({ + cache: CACHE_DIR, + registry: common.registry, + staleness: 600 + }) t.ok(stream, 'returned a stream') var results = [] stream.on('data', function (pkg) { @@ -184,7 +196,11 @@ test('createEntryStream merged stream', function (t) { test('allPackageMetadata no sources', function (t) { setup() server.get('/-/all').once().reply(404, {}) - var stream = allPackageMetadata(600) + var stream = allPackageMetadata({ + cache: CACHE_DIR, + registry: common.registry, + staleness: 600 + }) ms.finished(stream, function (err) { t.ok(err, 'no sources, got an error') t.match(err.message, /No search sources available/, 'useful error message') diff --git a/test/tap/search.all-package-search.js b/test/tap/search.all-package-search.js index c70f4f8e7ef07..51c1ffcf90157 100644 --- a/test/tap/search.all-package-search.js +++ b/test/tap/search.all-package-search.js @@ -1,21 +1,25 @@ -var path = require('path') -var mkdirp = require('mkdirp') -var mr = require('npm-registry-mock') -var osenv = require('osenv') -var rimraf = require('rimraf') -var cacheFile = require('npm-cache-filename') -var test = require('tap').test -var Tacks = require('tacks') -var File = Tacks.File +'use strict' -var common = require('../common-tap.js') +const cacheFile = require('npm-cache-filename') +const mkdirp = require('mkdirp') +const mr = require('npm-registry-mock') +const osenv = require('osenv') +const path = require('path') +const qs = require('querystring') +const rimraf = require('rimraf') +const Tacks = require('tacks') +const test = require('tap').test -var PKG_DIR = path.resolve(__dirname, 'search') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') -var cacheBase = cacheFile(CACHE_DIR)(common.registry + '/-/all') -var cachePath = path.join(cacheBase, '.cache.json') +const {File} = Tacks -var server +const common = require('../common-tap.js') + +const PKG_DIR = path.resolve(__dirname, 'search') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache') +const cacheBase = cacheFile(CACHE_DIR)(common.registry + '/-/all') +const cachePath = path.join(cacheBase, '.cache.json') + +let server test('setup', function (t) { mr({port: common.port, throwOnUnmatched: true}, function (err, s) { @@ -26,7 +30,7 @@ test('setup', function (t) { }) }) -var searches = [ +const searches = [ { term: 'cool', description: 'non-regex search', @@ -139,19 +143,26 @@ var searches = [ searches.forEach(function (search) { test(search.description, function (t) { setup() - server.get('/-/v1/search?text=' + encodeURIComponent(search.term) + '&size=20').once().reply(404, {}) - var now = Date.now() - var cacheContents = { + const query = qs.stringify({ + text: search.term, + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(404, {}) + const now = Date.now() + const cacheContents = { '_updated': now, bar: { name: 'bar', version: '1.0.0' }, cool: { name: 'cool', version: '5.0.0' }, foo: { name: 'foo', version: '2.0.0' }, other: { name: 'other', version: '1.0.0' } } - for (var k in search.inject) { + for (let k in search.inject) { cacheContents[k] = search.inject[k] } - var fixture = new Tacks(File(cacheContents)) + const fixture = new Tacks(File(cacheContents)) fixture.create(cachePath) common.npm([ 'search', search.term, @@ -167,12 +178,12 @@ searches.forEach(function (search) { t.equal(code, 0, 'search finished successfully') t.ifErr(err, 'search finished successfully') // \033 == \u001B - var markStart = '\u001B\\[[0-9][0-9]m' - var markEnd = '\u001B\\[0m' + const markStart = '\u001B\\[[0-9][0-9]m' + const markEnd = '\u001B\\[0m' - var re = new RegExp(markStart + '.*?' + markEnd) + const re = new RegExp(markStart + '.*?' + markEnd) - var cnt = stdout.search(re) + const cnt = stdout.search(re) t.equal( cnt, search.location, diff --git a/test/tap/search.esearch.js b/test/tap/search.esearch.js deleted file mode 100644 index d892aec95759c..0000000000000 --- a/test/tap/search.esearch.js +++ /dev/null @@ -1,192 +0,0 @@ -var common = require('../common-tap.js') -var finished = require('mississippi').finished -var mkdirp = require('mkdirp') -var mr = require('npm-registry-mock') -var npm = require('../../') -var osenv = require('osenv') -var path = require('path') -var rimraf = require('rimraf') -var test = require('tap').test - -var SEARCH = '/-/v1/search' -var PKG_DIR = path.resolve(__dirname, 'create-entry-update-stream') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') - -var esearch = require('../../lib/search/esearch') - -var server - -function setup () { - cleanup() - mkdirp.sync(CACHE_DIR) - process.chdir(CACHE_DIR) -} - -function cleanup () { - process.chdir(osenv.tmpdir()) - rimraf.sync(PKG_DIR) -} - -test('setup', function (t) { - mr({port: common.port, throwOnUnmatched: true}, function (err, s) { - t.ifError(err, 'registry mocked successfully') - npm.load({ cache: CACHE_DIR, registry: common.registry }, function (err) { - t.ifError(err, 'npm loaded successfully') - server = s - t.pass('all set up') - t.done() - }) - }) -}) - -test('basic test', function (t) { - setup() - server.get(SEARCH + '?text=oo&size=1').once().reply(200, { - objects: [ - { package: { name: 'cool', version: '1.0.0' } }, - { package: { name: 'foo', version: '2.0.0' } } - ] - }) - var results = [] - var s = esearch({ - include: ['oo'], - limit: 1 - }) - t.ok(s, 'got a stream') - s.on('data', function (d) { - results.push(d) - }) - finished(s, function (err) { - if (err) { throw err } - t.ok(true, 'stream finished without error') - t.deepEquals(results, [{ - name: 'cool', - version: '1.0.0', - description: null, - maintainers: null, - keywords: null, - date: null - }, { - name: 'foo', - version: '2.0.0', - description: null, - maintainers: null, - keywords: null, - date: null - }]) - server.done() - t.done() - }) -}) - -test('only returns certain fields for each package', function (t) { - setup() - var date = new Date() - server.get(SEARCH + '?text=oo&size=1').once().reply(200, { - objects: [{ - package: { - name: 'cool', - version: '1.0.0', - description: 'desc', - maintainers: [ - {username: 'x', email: 'a@b.c'}, - {username: 'y', email: 'c@b.a'} - ], - keywords: ['a', 'b', 'c'], - date: date.toISOString(), - extra: 'lol' - } - }] - }) - var results = [] - var s = esearch({ - include: ['oo'], - limit: 1 - }) - t.ok(s, 'got a stream') - s.on('data', function (d) { - results.push(d) - }) - finished(s, function (err) { - if (err) { throw err } - t.ok(true, 'stream finished without error') - t.deepEquals(results, [{ - name: 'cool', - version: '1.0.0', - description: 'desc', - maintainers: [ - {username: 'x', email: 'a@b.c'}, - {username: 'y', email: 'c@b.a'} - ], - keywords: ['a', 'b', 'c'], - date: date - }]) - server.done() - t.done() - }) -}) - -test('accepts a limit option', function (t) { - setup() - server.get(SEARCH + '?text=oo&size=3').once().reply(200, { - objects: [ - { package: { name: 'cool', version: '1.0.0' } }, - { package: { name: 'cool', version: '1.0.0' } } - ] - }) - var results = 0 - var s = esearch({ - include: ['oo'], - limit: 3 - }) - s.on('data', function () { results++ }) - finished(s, function (err) { - if (err) { throw err } - t.ok(true, 'request sent with correct size') - t.equal(results, 2, 'behaves fine with fewer results than size') - server.done() - t.done() - }) -}) - -test('passes foo:bar syntax params directly', function (t) { - setup() - server.get(SEARCH + '?text=foo%3Abar&size=1').once().reply(200, { - objects: [] - }) - var s = esearch({ - include: ['foo:bar'], - limit: 1 - }) - s.on('data', function () {}) - finished(s, function (err) { - if (err) { throw err } - t.ok(true, 'request sent with correct params') - server.done() - t.done() - }) -}) - -test('space-separates and URI-encodes multiple search params', function (t) { - setup() - server.get(SEARCH + '?text=foo%20bar%3Abaz%20quux%3F%3D&size=1').once().reply(200, { - objects: [] - }) - var s = esearch({ - include: ['foo', 'bar:baz', 'quux?='], - limit: 1 - }) - s.on('data', function () {}) - finished(s, function (err) { - if (err) { throw err } - t.ok(true, 'request sent with correct params') - server.done() - t.done() - }) -}) - -test('cleanup', function (t) { - server.close() - cleanup() - t.done() -}) diff --git a/test/tap/search.js b/test/tap/search.js index df7ff0fe375b7..bbd293c3a1a3f 100644 --- a/test/tap/search.js +++ b/test/tap/search.js @@ -1,21 +1,25 @@ -var path = require('path') -var mkdirp = require('mkdirp') -var mr = require('npm-registry-mock') -var osenv = require('osenv') -var rimraf = require('rimraf') -var cacheFile = require('npm-cache-filename') -var test = require('tap').test -var Tacks = require('tacks') -var File = Tacks.File +'use strict' -var common = require('../common-tap.js') +const cacheFile = require('npm-cache-filename') +const mkdirp = require('mkdirp') +const mr = require('npm-registry-mock') +const osenv = require('osenv') +const path = require('path') +const qs = require('querystring') +const rimraf = require('rimraf') +const test = require('tap').test -var PKG_DIR = path.resolve(__dirname, 'search') -var CACHE_DIR = path.resolve(PKG_DIR, 'cache') -var cacheBase = cacheFile(CACHE_DIR)(common.registry + '/-/all') -var cachePath = path.join(cacheBase, '.cache.json') +const Tacks = require('tacks') +const File = Tacks.File -var server +const common = require('../common-tap.js') + +const PKG_DIR = path.resolve(__dirname, 'search') +const CACHE_DIR = path.resolve(PKG_DIR, 'cache') +const cacheBase = cacheFile(CACHE_DIR)(common.registry + '/-/all') +const cachePath = path.join(cacheBase, '.cache.json') + +let server test('setup', function (t) { mr({port: common.port, throwOnUnmatched: true}, function (err, s) { @@ -28,7 +32,14 @@ test('setup', function (t) { test('notifies when there are no results', function (t) { setup() - server.get('/-/v1/search?text=none&size=20').once().reply(200, { + const query = qs.stringify({ + text: 'none', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(200, { objects: [] }) common.npm([ @@ -46,10 +57,17 @@ test('notifies when there are no results', function (t) { test('spits out a useful error when no cache nor network', function (t) { setup() - server.get('/-/v1/search?text=foo&size=20').once().reply(404, {}) + const query = qs.stringify({ + text: 'foo', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(404, {}) server.get('/-/all').many().reply(404, {}) - var cacheContents = {} - var fixture = new Tacks(File(cacheContents)) + const cacheContents = {} + const fixture = new Tacks(File(cacheContents)) fixture.create(cachePath) common.npm([ 'search', 'foo', @@ -70,7 +88,14 @@ test('spits out a useful error when no cache nor network', function (t) { test('can switch to JSON mode', function (t) { setup() - server.get('/-/v1/search?text=oo&size=20').once().reply(200, { + const query = qs.stringify({ + text: 'oo', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(200, { objects: [ { package: { name: 'cool', version: '1.0.0' } }, { package: { name: 'foo', version: '2.0.0' } } @@ -86,9 +111,15 @@ test('can switch to JSON mode', function (t) { if (err) throw err t.equal(stderr, '', 'no error output') t.equal(code, 0, 'search gives 0 error code even if no matches') - t.deepEquals(JSON.parse(stdout), [ - { name: 'cool', version: '1.0.0', date: null }, - { name: 'foo', version: '2.0.0', date: null } + t.similar(JSON.parse(stdout), [ + { + name: 'cool', + version: '1.0.0' + }, + { + name: 'foo', + version: '2.0.0' + } ], 'results returned as valid json') t.done() }) @@ -96,7 +127,14 @@ test('can switch to JSON mode', function (t) { test('JSON mode does not notify on empty', function (t) { setup() - server.get('/-/v1/search?text=oo&size=20').once().reply(200, { + const query = qs.stringify({ + text: 'oo', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(200, { objects: [] }) common.npm([ @@ -116,7 +154,14 @@ test('JSON mode does not notify on empty', function (t) { test('can switch to tab separated mode', function (t) { setup() - server.get('/-/v1/search?text=oo&size=20').once().reply(200, { + const query = qs.stringify({ + text: 'oo', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(200, { objects: [ { package: { name: 'cool', version: '1.0.0' } }, { package: { name: 'foo', description: 'this\thas\ttabs', version: '2.0.0' } } @@ -139,7 +184,14 @@ test('can switch to tab separated mode', function (t) { test('tab mode does not notify on empty', function (t) { setup() - server.get('/-/v1/search?text=oo&size=20').once().reply(200, { + const query = qs.stringify({ + text: 'oo', + size: 20, + quality: 0.65, + popularity: 0.98, + maintenance: 0.5 + }) + server.get(`/-/v1/search?${query}`).once().reply(200, { objects: [] }) common.npm([