Skip to content
This repository has been archived by the owner on Nov 30, 2023. It is now read-only.

Commit

Permalink
Support for threaded tweets
Browse files Browse the repository at this point in the history 
Close #84
  • Loading branch information
Alexander Sheiko committed Jun 5, 2023
1 parent 624adca commit 44d3873
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 42 deletions.
41 changes: 30 additions & 11 deletions timeline_v2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,16 @@ type entry struct {
Items []struct {
Item struct {
ItemContent struct {
TweetResults struct {
TweetDisplayType string `json:"tweetDisplayType"`
TweetResults struct {
Result result `json:"result"`
} `json:"tweet_results"`
} `json:"itemContent"`
} `json:"item"`
} `json:"items"`
ItemContent struct {
TweetResults struct {
TweetDisplayType string `json:"tweetDisplayType"`
TweetResults struct {
Result result `json:"result"`
} `json:"tweet_results"`
} `json:"itemContent"`
Expand Down Expand Up @@ -124,12 +126,18 @@ func (conversation *threadedConversation) parse() []*Tweet {
for _, entry := range instruction.Entries {
if entry.Content.ItemContent.TweetResults.Result.Typename == "Tweet" {
if tweet := entry.Content.ItemContent.TweetResults.Result.parse(); tweet != nil {
if entry.Content.ItemContent.TweetDisplayType == "SelfThread" {
tweet.IsSelfThread = true
}
tweets = append(tweets, tweet)
}
}
for _, item := range entry.Content.Items {
if item.Item.ItemContent.TweetResults.Result.Typename == "Tweet" {
if tweet := item.Item.ItemContent.TweetResults.Result.parse(); tweet != nil {
if item.Item.ItemContent.TweetDisplayType == "SelfThread" {
tweet.IsSelfThread = true
}
tweets = append(tweets, tweet)
}
}
Expand All @@ -145,6 +153,16 @@ func (conversation *threadedConversation) parse() []*Tweet {
}
}
}
if tweet.IsSelfThread && tweet.ConversationID == tweet.ID {
for _, childTweet := range tweets {
if childTweet.IsSelfThread && childTweet.ID != tweet.ID {
tweet.Thread = append(tweet.Thread, childTweet)
}
}
if len(tweet.Thread) == 0 {
tweet.IsSelfThread = false
}
}
}
return tweets
}
Expand All @@ -154,15 +172,16 @@ func parseLegacyTweet(user *legacyUser, tweet *legacyTweet) *Tweet {
name := user.Name
tweetID := tweet.IDStr
tw := &Tweet{
ID: tweetID,
Likes: tweet.FavoriteCount,
Name: name,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount,
Text: tweet.FullText,
UserID: tweet.UserIDStr,
Username: username,
ConversationID: tweet.ConversationIDStr,
ID: tweetID,
Likes: tweet.FavoriteCount,
Name: name,
PermanentURL: fmt.Sprintf("https://twitter.com/%s/status/%s", username, tweetID),
Replies: tweet.ReplyCount,
Retweets: tweet.RetweetCount,
Text: tweet.FullText,
UserID: tweet.UserIDStr,
Username: username,
}

tm, err := time.Parse(time.RubyDate, tweet.CreatedAt)
Expand Down
72 changes: 45 additions & 27 deletions tweets_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,17 @@ func TestGetTweets(t *testing.T) {

func TestGetTweet(t *testing.T) {
sample := twitterscraper.Tweet{
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>",
ID: "1328684389388185600",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
Photos: nil,
Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH",
TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)),
Timestamp: 1605618018,
UserID: "783214",
Username: "Twitter",
ConversationID: "1328684389388185600",
HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>",
ID: "1328684389388185600",
Name: "Twitter",
PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600",
Photos: nil,
Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH",
TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)),
Timestamp: 1605618018,
UserID: "783214",
Username: "Twitter",
Videos: []twitterscraper.Video{{
ID: "1328684333599756289",
Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg",
Expand Down Expand Up @@ -119,11 +120,12 @@ func TestTweetMentions(t *testing.T) {

func TestQuotedAndReply(t *testing.T) {
sample := &twitterscraper.Tweet{
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
ID: "1237110546383724547",
Likes: 485,
Name: "Vsauce2",
PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547",
ConversationID: "1237110546383724547",
HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --&gt; <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>",
ID: "1237110546383724547",
Likes: 485,
Name: "Vsauce2",
PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547",
Photos: []twitterscraper.Photo{{
ID: "1237110473486729218",
URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg",
Expand Down Expand Up @@ -164,18 +166,19 @@ func TestQuotedAndReply(t *testing.T) {
}
func TestRetweet(t *testing.T) {
sample := &twitterscraper.Tweet{
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
ID: "1359151057872580612",
Likes: 6683,
Name: "Twitter Together",
PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612",
Replies: 456,
Retweets: 1495,
Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)),
Timestamp: 1612881838,
UserID: "773578328498372608",
Username: "TwitterTogether",
ConversationID: "1359151057872580612",
HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
ID: "1359151057872580612",
Likes: 6683,
Name: "Twitter Together",
PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612",
Replies: 456,
Retweets: 1495,
Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.",
TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)),
Timestamp: 1612881838,
UserID: "773578328498372608",
Username: "TwitterTogether",
}
scraper := twitterscraper.New()
tweet, err := scraper.GetTweet("1362849141248974853")
Expand Down Expand Up @@ -217,3 +220,18 @@ func TestTweetViews(t *testing.T) {
}
}
}

func TestTweetThread(t *testing.T) {
scraper := twitterscraper.New()
tweet, err := scraper.GetTweet("1665602315745673217")
if err != nil {
t.Fatal(err)
} else {
if !tweet.IsSelfThread {
t.Error("IsSelfThread must be True")
}
if len(tweet.Thread) != 7 {
t.Error("Thread length must be 7")
}
}
}
12 changes: 8 additions & 4 deletions types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type (

// Tweet type.
Tweet struct {
ConversationID string
Hashtags []string
HTML string
ID string
Expand All @@ -34,6 +35,7 @@ type (
IsPin bool
IsReply bool
IsRetweet bool
IsSelfThread bool
Likes int
Name string
Mentions []Mention
Expand All @@ -47,6 +49,7 @@ type (
RetweetedStatus *Tweet
RetweetedStatusID string
Text string
Thread []*Tweet
TimeParsed time.Time
Timestamp int64
URLs []string
Expand All @@ -70,10 +73,11 @@ type (
}

legacyTweet struct {
CreatedAt string `json:"created_at"`
FavoriteCount int `json:"favorite_count"`
FullText string `json:"full_text"`
Entities struct {
ConversationIDStr string `json:"conversation_id_str"`
CreatedAt string `json:"created_at"`
FavoriteCount int `json:"favorite_count"`
FullText string `json:"full_text"`
Entities struct {
Hashtags []struct {
Text string `json:"text"`
} `json:"hashtags"`
Expand Down

0 comments on commit 44d3873

Please sign in to comment.