feat: apify get followers

Dockerfile

@@ -1,4 +1,4 @@
-ARG egover=1.6.0
+ARG egover=1.7.2
 ARG baseimage=ghcr.io/edgelesssys/ego-deploy:v${egover}
 ARG VERSION
 

Makefile

@@ -76,5 +76,8 @@ test-jobs: docker-build-test
 test-twitter: docker-build-test
 	@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) go test -v ./internal/jobs/twitter_test.go ./internal/jobs/jobs_suite_test.go
 
+test-web: docker-build-test
+	@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) go test -v ./internal/jobs/webscraper_test.go ./internal/jobs/jobs_suite_test.go
+
 test-telemetry: docker-build-test
 	@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) go test -v ./internal/jobs/telemetry_test.go ./internal/jobs/jobs_suite_test.go

README.md

@@ -58,6 +58,7 @@ The tee-worker requires various environment variables for operation. These shoul
 - `TWITTER_SKIP_LOGIN_VERIFICATION`: Set to `true` to skip Twitter's login verification step. This can help avoid rate limiting issues with Twitter's verify_credentials API endpoint when running multiple workers or processing large volumes of requests.
 - `TIKTOK_DEFAULT_LANGUAGE`: Default language for TikTok transcriptions (default: `eng-US`).
 - `TIKTOK_API_USER_AGENT`: User-Agent header for TikTok API requests (default: standard mobile browser user agent).
+- `APIFY_API_KEY`: API key for Apify Twitter scraping services. Required for `twitter-apify` job type and enables enhanced follower/following data collection.
 - `LISTEN_ADDRESS`: The address the service listens on (default: `:8080`).
 - `RESULT_CACHE_MAX_SIZE`: Maximum number of job results to keep in the result cache (default: `1000`).
 - `RESULT_CACHE_MAX_AGE_SECONDS`: Maximum age (in seconds) to keep a result in the cache (default: `600`).
@@ -72,11 +73,11 @@ The worker automatically detects and exposes capabilities based on available con
 **Core Services (Always Available):**
 
 1. **`web`** - Web scraping services
-   - **Sub-capabilities**: `["web-scraper"]`
+   - **Sub-capabilities**: `["scraper"]`
    - **Requirements**: None (always available)
 
 2. **`tiktok`** - TikTok video processing
-   - **Sub-capabilities**: `["tiktok-transcription"]`
+   - **Sub-capabilities**: `["transcription"]`
    - **Requirements**: None (always available)
 
 **Twitter Services (Configuration-Dependent):**
@@ -90,12 +91,17 @@ The worker automatically detects and exposes capabilities based on available con
    - **Requirements**: `TWITTER_API_KEYS` environment variable
 
 5. **`twitter`** - General Twitter scraping (uses best available auth)
-   - **Sub-capabilities**: Dynamic based on available authentication (same as credential or API depending on what's configured)
-   - **Requirements**: Either `TWITTER_ACCOUNTS` or `TWITTER_API_KEYS`
+   - **Sub-capabilities**: Dynamic based on available authentication (combines capabilities from credential, API, and Apify depending on what's configured)
+   - **Requirements**: Either `TWITTER_ACCOUNTS`, `TWITTER_API_KEYS`, or `APIFY_API_KEY`
+   - **Priority**: For follower/following operations: Apify > Credentials. For search operations: Credentials > API.
+
+6. **`twitter-apify`** - Twitter scraping using Apify's API (requires `APIFY_API_KEY`)
+   - **Sub-capabilities**: `["getfollowers", "getfollowing"]`
+   - **Requirements**: `APIFY_API_KEY` environment variable
 
 **Stats Service (Always Available):**
 
-6. **`telemetry`** - Worker monitoring and stats
+7. **`telemetry`** - Worker monitoring and stats
    - **Sub-capabilities**: `["telemetry"]`
    - **Requirements**: None (always available)
 
@@ -142,7 +148,7 @@ curl -s localhost:8080/job/result \
 
 All job types follow the same API flow above. Here are the available job types and their specific parameters:
 
-#### `web-scraper`
+#### `web`
 Scrapes content from web pages.
 
 **Parameters:**
@@ -151,8 +157,9 @@ Scrapes content from web pages.
 
 ```json
 {
-  "type": "web-scraper",
+  "type": "web",
   "arguments": {
+    "type": "scraper",
     "url": "https://www.google.com",
     "depth": 1
   }
@@ -185,8 +192,9 @@ Transcribes TikTok videos to text.
 
 ```json
 {
-  "type": "tiktok-transcription",
+  "type": "tiktok",
   "arguments": {
+    "type": "transcription",
     "video_url": "https://www.tiktok.com/@coachty23/video/7502100651397172526",
     "language": "eng-US"
   }
@@ -195,10 +203,11 @@ Transcribes TikTok videos to text.
 
 #### Twitter Job Types
 
-Twitter scraping is available through three job types:
-- `twitter-scraper`: Uses best available auth method (credential or API)
-- `twitter-credential-scraper`: Forces credential-based scraping (requires `TWITTER_ACCOUNTS`)
-- `twitter-api-scraper`: Forces API-based scraping (requires `TWITTER_API_KEYS`)
+Twitter scraping is available through four job types:
+- `twitter`: Uses best available auth method (credential, API, or Apify)
+- `twitter-credential`: Forces credential-based scraping (requires `TWITTER_ACCOUNTS`)
+- `twitter-api`: Forces API-based scraping (requires `TWITTER_API_KEYS`)
+- `twitter-apify`: Forces Apify-based scraping (requires `APIFY_API_KEY`)
 
 **Common Parameters:**
 - `type` (string, required): The operation type (see sub-capabilities below)
@@ -211,7 +220,7 @@ Twitter scraping is available through three job types:
 **`searchbyquery`** - Search tweets using Twitter query syntax
 ```json
 {
-  "type": "twitter-scraper",
+  "type": "twitter",
   "arguments": {
     "type": "searchbyquery",
     "query": "climate change",
@@ -223,7 +232,7 @@ Twitter scraping is available through three job types:
 **`searchbyfullarchive`** - Search full tweet archive (requires elevated API key for API-based scraping)
 ```json
 {
-  "type": "twitter-api-scraper",
+  "type": "twitter-api",
   "arguments": {
     "type": "searchbyfullarchive",
     "query": "NASA",
@@ -235,7 +244,7 @@ Twitter scraping is available through three job types:
 **`getbyid`** - Get specific tweet by ID
 ```json
 {
-  "type": "twitter-scraper",
+  "type": "twitter",
   "arguments": {
     "type": "getbyid",
     "query": "1881258110712492142"
@@ -246,7 +255,7 @@ Twitter scraping is available through three job types:
 **`getreplies`** - Get replies to a specific tweet
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getreplies",
     "query": "1234567890",
@@ -258,7 +267,7 @@ Twitter scraping is available through three job types:
 **`getretweeters`** - Get users who retweeted a specific tweet
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getretweeters",
     "query": "1234567890",
@@ -272,7 +281,7 @@ Twitter scraping is available through three job types:
 **`gettweets`** - Get tweets from a user's timeline
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "gettweets",
     "query": "NASA",
@@ -284,7 +293,7 @@ Twitter scraping is available through three job types:
 **`getmedia`** - Get media (photos/videos) from a user
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getmedia",
     "query": "NASA",
@@ -296,7 +305,7 @@ Twitter scraping is available through three job types:
 **`gethometweets`** - Get authenticated user's home timeline (credential-based only)
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "gethometweets",
     "max_results": 30
@@ -307,7 +316,7 @@ Twitter scraping is available through three job types:
 **`getforyoutweets`** - Get "For You" timeline (credential-based only)
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getforyoutweets",
     "max_results": 25
@@ -320,7 +329,7 @@ Twitter scraping is available through three job types:
 **`searchbyprofile`** - Get user profile information
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "searchbyprofile",
     "query": "NASA_Marshall"
@@ -331,7 +340,7 @@ Twitter scraping is available through three job types:
 **`getprofilebyid`** - Get user profile by user ID
 ```json
 {
-  "type": "twitter-scraper",
+  "type": "twitter",
   "arguments": {
     "type": "getprofilebyid",
     "query": "44196397"
@@ -342,7 +351,7 @@ Twitter scraping is available through three job types:
 **`getfollowers`** - Get followers of a profile
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getfollowers",
     "query": "NASA",
@@ -351,10 +360,23 @@ Twitter scraping is available through three job types:
 }
 ```
 
+**`getfollowers`** (using Apify for enhanced data) - Get followers with detailed profile information
+```json
+{
+  "type": "twitter-apify",
+  "arguments": {
+    "type": "getfollowers",
+    "query": "NASA",
+    "max_results": 100,
+    "next_cursor": "optional_pagination_cursor"
+  }
+}
+```
+
 **`getfollowing`** - Get users that a profile is following
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "getfollowing",
     "query": "NASA",
@@ -363,18 +385,43 @@ Twitter scraping is available through three job types:
 }
 ```
 
+**`getfollowing`** (using Apify for enhanced data) - Get following with detailed profile information
+```json
+{
+  "type": "twitter-apify",
+  "arguments": {
+    "type": "getfollowing",
+    "query": "NASA",
+    "max_results": 100,
+    "next_cursor": "optional_pagination_cursor"
+  }
+}
+```
+
 ##### Other Operations
 
 **`gettrends`** - Get trending topics (no query required)
 ```json
 {
-  "type": "twitter-credential-scraper",
+  "type": "twitter-credential",
   "arguments": {
     "type": "gettrends"
   }
 }
 ```
 
+##### Return Types
+
+**Enhanced Profile Data with Apify**: When using `twitter-apify` for `getfollowers` or `getfollowing` operations, the response returns `ProfileResultApify` objects which include comprehensive profile information such as:
+- Basic profile data (ID, name, screen name, location, description)
+- Detailed follower/following counts and engagement metrics
+- Profile appearance settings and colors
+- Account verification and security status
+- Privacy and interaction settings
+- Business account information when available
+
+This enhanced data provides richer insights compared to standard credential or API-based profile results.
+
 ### Health Check Endpoints
 
 The service provides health check endpoints:

api/types/job.go

@@ -180,3 +180,25 @@ func (jc JobConfiguration) GetBool(key string, def bool) bool {
 	}
 	return def
 }
+
+// TwitterScraperConfig represents the configuration needed for Twitter scraping
+// This is defined here to avoid circular imports between api/types and internal/jobs
+type TwitterScraperConfig struct {
+	Accounts              []string
+	ApiKeys               []string
+	ApifyApiKey           string
+	DataDir               string
+	SkipLoginVerification bool
+}
+
+// GetTwitterConfig constructs a TwitterScraperConfig directly from the JobConfiguration
+// This eliminates the need for JSON marshaling/unmarshaling
+func (jc JobConfiguration) GetTwitterConfig() TwitterScraperConfig {
+	return TwitterScraperConfig{
+		Accounts:              jc.GetStringSlice("twitter_accounts", []string{}),
+		ApiKeys:               jc.GetStringSlice("twitter_api_keys", []string{}),
+		ApifyApiKey:           jc.GetString("apify_api_key", ""),
+		DataDir:               jc.GetString("data_dir", ""),
+		SkipLoginVerification: jc.GetBool("skip_login_verification", false),
+	}
+}

cmd/tee-worker/config.go

@@ -103,6 +103,15 @@ func readConfig() types.JobConfiguration {
 		jc["twitter_api_keys"] = []string{}
 	}
 
+	// Apify API key loading
+	apifyApiKey := os.Getenv("APIFY_API_KEY")
+	if apifyApiKey != "" {
+		logrus.Info("Apify API key found")
+		jc["apify_api_key"] = apifyApiKey
+	} else {
+		jc["apify_api_key"] = ""
+	}
+
 	tikTokLang := os.Getenv("TIKTOK_DEFAULT_LANGUAGE")
 	if tikTokLang == "" {
 		tikTokLang = "eng-US"