feat: supports apify work in stats

internal/jobs/stats/stats.go

@@ -19,6 +19,7 @@ const (
 	TwitterScrapes             StatType = "twitter_scrapes"
 	TwitterTweets              StatType = "twitter_returned_tweets"
 	TwitterProfiles            StatType = "twitter_returned_profiles"
+	TwitterFollowers           StatType = "twitter_returned_followers"
 	TwitterOther               StatType = "twitter_returned_other"
 	TwitterErrors              StatType = "twitter_errors"
 	TwitterAuthErrors          StatType = "twitter_auth_errors"
@@ -29,7 +30,7 @@ const (
 	WebInvalid                 StatType = "web_invalid"
 	TikTokTranscriptionSuccess StatType = "tiktok_transcription_success"
 	TikTokTranscriptionErrors  StatType = "tiktok_transcription_errors"
-	// TODO: Should we add stats for calls to each of the Twitter job types?
+	// TODO: Should we add stats for calls to each of the Twitter capabilities to decouple business / scoring logic?
 )
 
 // AddStat is the struct used in the rest of the tee-worker for sending statistics

internal/jobs/twitter.go

@@ -893,7 +893,7 @@ func (ts *TwitterScraper) getFollowersApify(j types.Job, username string, maxRes
 		return nil, "", err
 	}
 
-	ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(followers)))
+	ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(followers)))
 	return followers, nextCursor, nil
 }
 
@@ -911,7 +911,7 @@ func (ts *TwitterScraper) getFollowingApify(j types.Job, username string, maxRes
 		return nil, "", err
 	}
 
-	ts.statsCollector.Add(j.WorkerID, stats.TwitterProfiles, uint(len(following)))
+	ts.statsCollector.Add(j.WorkerID, stats.TwitterFollowers, uint(len(following)))
 	return following, nextCursor, nil
 }
 

internal/jobs/twitter_test.go

@@ -796,15 +796,18 @@ var _ = Describe("Twitter Scraper", func() {
 				"apify_api_key": apifyApiKey,
 				"data_dir":      tempDir,
 			}, statsCollector)
-			res, err := scraper.ExecuteJob(types.Job{
+
+			j := types.Job{
 				Type: teetypes.TwitterApifyJob,
 				Arguments: map[string]interface{}{
 					"type":        teetypes.CapGetFollowers,
 					"query":       "elonmusk",
 					"max_results": 200,
 				},
 				Timeout: 60 * time.Second,
-			})
+			}
+
+			res, err := scraper.ExecuteJob(j)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(res.Error).To(BeEmpty())
 
@@ -813,6 +816,7 @@ var _ = Describe("Twitter Scraper", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(followers).ToNot(BeEmpty())
 			Expect(followers[0].ScreenName).ToNot(BeEmpty())
+			Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(followers))))
 		})
 
 		It("should use Apify for twitter-apify with getfollowing", func() {
@@ -823,15 +827,18 @@ var _ = Describe("Twitter Scraper", func() {
 				"apify_api_key": apifyApiKey,
 				"data_dir":      tempDir,
 			}, statsCollector)
-			res, err := scraper.ExecuteJob(types.Job{
+
+			j := types.Job{
 				Type: teetypes.TwitterApifyJob,
 				Arguments: map[string]interface{}{
 					"type":        teetypes.CapGetFollowing,
 					"query":       "elonmusk",
 					"max_results": 200,
 				},
 				Timeout: 60 * time.Second,
-			})
+			}
+
+			res, err := scraper.ExecuteJob(j)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(res.Error).To(BeEmpty())
 
@@ -840,6 +847,7 @@ var _ = Describe("Twitter Scraper", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(following).ToNot(BeEmpty())
 			Expect(following[0].ScreenName).ToNot(BeEmpty())
+			Expect(statsCollector.Stats.Stats[j.WorkerID][stats.TwitterFollowers]).To(BeNumerically("==", uint(len(following))))
 		})
 
 		It("should prioritize Apify for general twitter job with getfollowers", func() {