From b9e914b5564dc1b5ae93f7725634186732bf95ee Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 14 Aug 2025 19:07:44 +0200 Subject: [PATCH 1/2] feat: supports apify work in stats --- internal/jobs/stats/stats.go | 3 ++- internal/jobs/twitter.go | 4 ++-- internal/jobs/twitter_test.go | 17 +++++++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/jobs/stats/stats.go b/internal/jobs/stats/stats.go index 33cb3d7a..5eac0bd6 100644 --- a/internal/jobs/stats/stats.go +++ b/internal/jobs/stats/stats.go @@ -19,6 +19,7 @@ const ( TwitterScrapes StatType = "twitter_scrapes" TwitterTweets StatType = "twitter_returned_tweets" TwitterProfiles StatType = "twitter_returned_profiles" + TwitterFollowers StatType = "twitter_returned_followers" TwitterOther StatType = "twitter_returned_other" TwitterErrors StatType = "twitter_errors" TwitterAuthErrors StatType = "twitter_auth_errors" @@ -29,7 +30,7 @@ const ( WebInvalid StatType = "web_invalid" TikTokTranscriptionSuccess StatType = "tiktok_transcription_success" TikTokTranscriptionErrors StatType = "tiktok_transcription_errors" - // TODO: Should we add stats for calls to each of the Twitter job types? + // TODO: Should we add stats for calls to each of the Twitter capabilities to decouple business / scoring logic? ) // AddStat is the struct used in the rest of the tee-worker for sending statistics diff --git a/internal/jobs/twitter.go b/internal/jobs/twitter.go index 3d5a345e..c9874c5a 100644 --- a/internal/jobs/twitter.go +++ b/internal/jobs/twitter.go @@ -893,7 +893,7 @@ func (ts *TwitterScraper) getFollowersApify(j types.Job, username string, maxRes return nil, "", err } - ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(followers))) + ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(followers))) return followers, nextCursor, nil } @@ -911,7 +911,7 @@ func (ts *TwitterScraper) getFollowingApify(j types.Job, username string, maxRes return nil, "", err } - ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(following))) + ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(following))) return following, nextCursor, nil } diff --git a/internal/jobs/twitter_test.go b/internal/jobs/twitter_test.go index 1da29053..7256680b 100644 --- a/internal/jobs/twitter_test.go +++ b/internal/jobs/twitter_test.go @@ -796,7 +796,8 @@ var _ = Describe("Twitter Scraper", func() { "apify_api_key": apifyApiKey, "data_dir": tempDir, }, statsCollector) - res, err := scraper.ExecuteJob(types.Job{ + + j := types.Job{ Type: teetypes.TwitterApifyJob, Arguments: map[string]interface{}{ "type": teetypes.CapGetFollowers, @@ -804,7 +805,9 @@ var _ = Describe("Twitter Scraper", func() { "max_results": 200, }, Timeout: 60 * time.Second, - }) + } + + res, err := scraper.ExecuteJob(j) Expect(err).NotTo(HaveOccurred()) Expect(res.Error).To(BeEmpty()) @@ -813,6 +816,8 @@ var _ = Describe("Twitter Scraper", func() { Expect(err).NotTo(HaveOccurred()) Expect(followers).ToNot(BeEmpty()) Expect(followers[0].ScreenName).ToNot(BeEmpty()) + Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(followers)))) + }) It("should use Apify for twitter-apify with getfollowing", func() { @@ -823,7 +828,8 @@ var _ = Describe("Twitter Scraper", func() { "apify_api_key": apifyApiKey, "data_dir": tempDir, }, statsCollector) - res, err := scraper.ExecuteJob(types.Job{ + + j := types.Job{ Type: teetypes.TwitterApifyJob, Arguments: map[string]interface{}{ "type": teetypes.CapGetFollowing, @@ -831,7 +837,9 @@ var _ = Describe("Twitter Scraper", func() { "max_results": 200, }, Timeout: 60 * time.Second, - }) + } + + res, err := scraper.ExecuteJob(j) Expect(err).NotTo(HaveOccurred()) Expect(res.Error).To(BeEmpty()) @@ -840,6 +848,7 @@ var _ = Describe("Twitter Scraper", func() { Expect(err).NotTo(HaveOccurred()) Expect(following).ToNot(BeEmpty()) Expect(following[0].ScreenName).ToNot(BeEmpty()) + Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(following)))) }) It("should prioritize Apify for general twitter job with getfollowers", func() { From d915c2ccbcba7b7f55748ebc71205b6ca2a88aac Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 14 Aug 2025 19:24:26 +0200 Subject: [PATCH 2/2] chore: cleanup spacing --- internal/jobs/twitter_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/jobs/twitter_test.go b/internal/jobs/twitter_test.go index 7256680b..a8bde3ac 100644 --- a/internal/jobs/twitter_test.go +++ b/internal/jobs/twitter_test.go @@ -817,7 +817,6 @@ var _ = Describe("Twitter Scraper", func() { Expect(followers).ToNot(BeEmpty()) Expect(followers[0].ScreenName).ToNot(BeEmpty()) Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(followers)))) - }) It("should use Apify for twitter-apify with getfollowing", func() {