diff --git a/internal/jobs/stats/stats.go b/internal/jobs/stats/stats.go index 33cb3d7a..5eac0bd6 100644 --- a/internal/jobs/stats/stats.go +++ b/internal/jobs/stats/stats.go @@ -19,6 +19,7 @@ const ( TwitterScrapes StatType = "twitter_scrapes" TwitterTweets StatType = "twitter_returned_tweets" TwitterProfiles StatType = "twitter_returned_profiles" + TwitterFollowers StatType = "twitter_returned_followers" TwitterOther StatType = "twitter_returned_other" TwitterErrors StatType = "twitter_errors" TwitterAuthErrors StatType = "twitter_auth_errors" @@ -29,7 +30,7 @@ const ( WebInvalid StatType = "web_invalid" TikTokTranscriptionSuccess StatType = "tiktok_transcription_success" TikTokTranscriptionErrors StatType = "tiktok_transcription_errors" - // TODO: Should we add stats for calls to each of the Twitter job types? + // TODO: Should we add stats for calls to each of the Twitter capabilities to decouple business / scoring logic? ) // AddStat is the struct used in the rest of the tee-worker for sending statistics diff --git a/internal/jobs/twitter.go b/internal/jobs/twitter.go index 3d5a345e..c9874c5a 100644 --- a/internal/jobs/twitter.go +++ b/internal/jobs/twitter.go @@ -893,7 +893,7 @@ func (ts *TwitterScraper) getFollowersApify(j types.Job, username string, maxRes return nil, "", err } - ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(followers))) + ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(followers))) return followers, nextCursor, nil } @@ -911,7 +911,7 @@ func (ts *TwitterScraper) getFollowingApify(j types.Job, username string, maxRes return nil, "", err } - ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(following))) + ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(following))) return following, nextCursor, nil } diff --git a/internal/jobs/twitter_test.go b/internal/jobs/twitter_test.go index 1da29053..a8bde3ac 100644 --- a/internal/jobs/twitter_test.go +++ b/internal/jobs/twitter_test.go @@ -796,7 +796,8 @@ var _ = Describe("Twitter Scraper", func() { "apify_api_key": apifyApiKey, "data_dir": tempDir, }, statsCollector) - res, err := scraper.ExecuteJob(types.Job{ + + j := types.Job{ Type: teetypes.TwitterApifyJob, Arguments: map[string]interface{}{ "type": teetypes.CapGetFollowers, @@ -804,7 +805,9 @@ var _ = Describe("Twitter Scraper", func() { "max_results": 200, }, Timeout: 60 * time.Second, - }) + } + + res, err := scraper.ExecuteJob(j) Expect(err).NotTo(HaveOccurred()) Expect(res.Error).To(BeEmpty()) @@ -813,6 +816,7 @@ var _ = Describe("Twitter Scraper", func() { Expect(err).NotTo(HaveOccurred()) Expect(followers).ToNot(BeEmpty()) Expect(followers[0].ScreenName).ToNot(BeEmpty()) + Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(followers)))) }) It("should use Apify for twitter-apify with getfollowing", func() { @@ -823,7 +827,8 @@ var _ = Describe("Twitter Scraper", func() { "apify_api_key": apifyApiKey, "data_dir": tempDir, }, statsCollector) - res, err := scraper.ExecuteJob(types.Job{ + + j := types.Job{ Type: teetypes.TwitterApifyJob, Arguments: map[string]interface{}{ "type": teetypes.CapGetFollowing, @@ -831,7 +836,9 @@ var _ = Describe("Twitter Scraper", func() { "max_results": 200, }, Timeout: 60 * time.Second, - }) + } + + res, err := scraper.ExecuteJob(j) Expect(err).NotTo(HaveOccurred()) Expect(res.Error).To(BeEmpty()) @@ -840,6 +847,7 @@ var _ = Describe("Twitter Scraper", func() { Expect(err).NotTo(HaveOccurred()) Expect(following).ToNot(BeEmpty()) Expect(following[0].ScreenName).ToNot(BeEmpty()) + Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(following)))) }) It("should prioritize Apify for general twitter job with getfollowers", func() {