Skip to content
This repository has been archived by the owner on May 12, 2020. It is now read-only.

Commit

Permalink
Add the ability to upload an individual dat file
Browse files Browse the repository at this point in the history
  • Loading branch information
bbondy committed Oct 31, 2016
1 parent 9226f1f commit 412b665
Showing 1 changed file with 18 additions and 6 deletions.
24 changes: 18 additions & 6 deletions scripts/uploadDataFiles.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
const fs = require('fs')
const s3 = require('s3')
const commander = require('commander')
const path = require('path')

const client = s3.createClient({
maxAsyncS3: 20,
s3RetryCount: 3,
Expand All @@ -10,10 +13,10 @@ const client = s3.createClient({
s3Options: {}
})

const uploadFile = (filename) => {
const uploadFile = (filePath, filename) => {
return new Promise((resolve, reject) => {
var params = {
localFile: `out/${filename}`,
localFile: filePath,
// See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
s3Params: {
Bucket: 'adblock-data',
Expand All @@ -34,9 +37,18 @@ const uploadFile = (filename) => {
})
}

commander
.option('-d, --dat [dat]', 'file path of the adblock .dat file to upload')
.parse(process.argv)

// Queue up all the uploads one at a time to easily spot errors
let p = Promise.resolve()
const dataFilenames = fs.readdirSync('out')
dataFilenames.forEach((filename) => {
p = p.then(uploadFile.bind(null, filename))
})

if (commander.dat) {
p = p.then(uploadFile.bind(null, commander.dat, path.basename(commander.dat)))
} else {
const dataFilenames = fs.readdirSync('out')
dataFilenames.forEach((filename) => {
p = p.then(uploadFile.bind(null, `out/${filename}`, filename))
})
}

1 comment on commit 412b665

@bbondy
Copy link
Member Author

@bbondy bbondy commented on 412b665 Oct 31, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.