fix(timeline): API for replies now requires login; needs refactor

src/bin/scrape-twitter-timeline.js

@@ -5,9 +5,12 @@ const pump = require('pump')
 const TimelineStream = require('../lib/timeline-stream')
 const cliUtils = require('../lib/cli-utils')
 
+const SCRAPE_TWITTER_CONFIG = cliUtils.SCRAPE_TWITTER_CONFIG
+const env = cliUtils.getEnv()
+
 const cli = meow(`
   Usage
-    $ scrape-twitter timeline [--count=<count>] <username>
+    $ TWITTER_USERNAME=jack TWITTER_PASSWORD=p4ssw0rd scrape-twitter timeline [--count=<count>] <username>
 
   Options
     --with-retweets, -t   Include retweets
@@ -20,12 +23,18 @@ const cli = meow(`
 
 if (cli.input.length === 0) {
   cli.showHelp()
+} else if (cli.flags.withReplies && (!env.TWITTER_USERNAME || !env.TWITTER_PASSWORD)) {
+  console.log('Please ensure that the environment variables TWITTER_USERNAME and TWITTER_PASSWORD are set.')
+  console.log()
+  console.log(`Environment variables can be set within the dotenv file: ${SCRAPE_TWITTER_CONFIG}`)
+  process.exit(1)
 } else {
   const username = cliUtils.parseUsername(cli.input[0])
   const tweets = new TimelineStream(username, {
     retweets: cli.flags.withRetweets || false,
     replies: cli.flags.withReplies || false,
-    count: cli.flags.count
+    count: cli.flags.count,
+    env
   })
   pump(tweets, JSONStream.stringify('[\n', ',\n', '\n]\n'), process.stdout, cliUtils.handleError(process.exit.bind(process)))
 }

src/lib/timeline-stream.js

@@ -1,21 +1,26 @@
 const Readable = require('readable-stream/readable')
 const debug = require('debug')('scrape-twitter:timeline-stream')
 
+const twitterLogin = require('./twitter-login')
 const twitterQuery = require('./twitter-query')
 
+const loginWhenReplies = (replies, env = {}) =>
+  replies ? twitterLogin(env) : Promise.resolve()
+
 class TimelineStream extends Readable {
 
   isLocked = false
 
   _numberOfTweetsRead = 0
   _lastReadTweetId = undefined
 
-  constructor (username, { retweets, replies, count } = {}) {
+  constructor (username, { retweets, replies, count, env } = {}) {
     super({ objectMode: true })
     this.username = username
     this.retweets = retweets == null ? false : retweets
     this.replies = replies == null ? false : replies
     this.count = count
+    this.env = env
     debug(`TimelineStream for ${this.username} created with`, { retweets: this.retweets, replies: this.replies, count: this.count })
   }
 
@@ -39,7 +44,13 @@ class TimelineStream extends Readable {
     debug('TimelineStream is now locked')
 
     debug(`TimelineStream reads timeline${this._lastReadTweetId ? ` from tweet ${this._lastReadTweetId} onwards` : ''}`)
-    twitterQuery.getUserTimeline(this.username, this._lastReadTweetId, { replies: this.replies })
+
+    loginWhenReplies(this.replies, this.env).then(() =>
+      twitterQuery.getUserTimeline(
+        this.username,
+        this._lastReadTweetId,
+        { replies: this.replies }
+      )
       .then(tweets => {
         let lastReadTweetId
         for (const tweet of tweets) {
@@ -89,6 +100,8 @@ class TimelineStream extends Readable {
         }
       })
       .catch(err => this.emit('error', err))
+    )
+    .catch(err => this.emit('error', err))
   }
 
 }

src/lib/twitter-query.js

@@ -12,7 +12,7 @@ const getUserTimeline = (username, startingId, { replies = false }) => {
     'include_entities': '1',
     'max_position': startingId
   }
-  return query(url, options)
+  return query(url, options, replies ? fetchWithCookie : undefined)
     .then(toCheerio)
     .then(parser.toTweets)
 }

test/index.spec.js

@@ -205,7 +205,6 @@ test('ListStream should emit a particular set of tweets', () => {
     isPinned: expect.any(Boolean),
     isReplyTo: expect.any(Boolean),
     isRetweet: expect.any(Boolean),
-    quote: expect.anything(),
     userMentions: expect.any(Array),
     urls: expect.any(Array),
     hashtags: expect.any(Array),