From 040f616543fe161f4cc4a5922c72b588c5c61c87 Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 23 Oct 2025 08:28:40 +0200 Subject: [PATCH 1/4] fix: handle apify returns --- internal/jobs/twitter.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/jobs/twitter.go b/internal/jobs/twitter.go index 4ac97dfe..8c3e7e79 100644 --- a/internal/jobs/twitter.go +++ b/internal/jobs/twitter.go @@ -739,6 +739,12 @@ func (ts *TwitterScraper) ExecuteJob(j types.Job) (types.JobResult, error) { // Validate the result based on operation type switch { + case args.Type == types.CapGetFollowers || args.Type == types.CapGetFollowing: + var results []*types.ProfileResultApify + if err := jobResult.Unmarshal(&results); err != nil { + logrus.Errorf("Error while unmarshalling followers/following result for job ID %s, type %s: %v", j.UUID, j.Type, err) + return types.JobResult{Error: "error unmarshalling followers/following result for final validation"}, err + } case args.IsSingleTweetOperation(): var result *types.TweetResult if err := jobResult.Unmarshal(&result); err != nil { From 9340da400d28e94ba1ec50f832bcd1f43e3df89f Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 23 Oct 2025 08:51:25 +0200 Subject: [PATCH 2/4] fix: validation --- api/params/base.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/api/params/base.go b/api/params/base.go index 649639ea..18b13927 100644 --- a/api/params/base.go +++ b/api/params/base.go @@ -31,7 +31,21 @@ type Params[T base.JobArgument] struct { } func (p Params[T]) Validate(_ *types.SearchConfig) error { - return p.Args.Validate() + // Convert Args (generic T) to map[string]any + jsonData, err := json.Marshal(p.Args) + if err != nil { + return err + } + var argsMap map[string]any + if err := json.Unmarshal(jsonData, &argsMap); err != nil { + return err + } + + // Force typed unmarshal by job type which triggers defaults + Validate() + if _, err := args.UnmarshalJobArguments(p.JobType, argsMap); err != nil { + return err + } + return nil } func (p Params[T]) Type() types.JobType { @@ -60,8 +74,9 @@ func (l Params[T]) Arguments(cfg *types.SearchConfig) map[string]any { // Use UnmarshalJobArguments to get properly typed arguments with correct type ja, err := args.UnmarshalJobArguments(l.JobType, argsMap) if err != nil { - // Fallback to original args if unmarshaling fails - return argsMap + // Validation should have already occurred in Validate(); avoid silently + // proceeding with potentially invalid arguments here. + return nil } // Convert ja to map[string]any From f31093625677de52183424defacbd734bf398ad0 Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 23 Oct 2025 09:03:32 +0200 Subject: [PATCH 3/4] chore: fix error messaging tree --- api/args/unmarshaller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/args/unmarshaller.go b/api/args/unmarshaller.go index 16d2a89a..41622540 100644 --- a/api/args/unmarshaller.go +++ b/api/args/unmarshaller.go @@ -55,7 +55,7 @@ func UnmarshalJobArguments(jobType types.JobType, args Args) (base.JobArgument, func unmarshalWebArguments(args Args) (*web.ScraperArguments, error) { webArgs := &web.ScraperArguments{} if err := unmarshalToStruct(args, webArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, fmt.Errorf("%w", err) } return webArgs, nil } @@ -63,25 +63,25 @@ func unmarshalWebArguments(args Args) (*web.ScraperArguments, error) { func unmarshalTikTokArguments(args Args) (base.JobArgument, error) { minimal := base.Arguments{} if err := unmarshalToStruct(args, &minimal); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, fmt.Errorf("%w", err) } switch minimal.Type { case types.CapSearchByQuery: searchArgs := &tiktok.QueryArguments{} if err := unmarshalToStruct(args, searchArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, fmt.Errorf("%w", err) } return searchArgs, nil case types.CapSearchByTrending: searchArgs := &tiktok.TrendingArguments{} if err := unmarshalToStruct(args, searchArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, fmt.Errorf("%w", err) } return searchArgs, nil case types.CapTranscription: transcriptionArgs := &tiktok.TranscriptionArguments{} if err := unmarshalToStruct(args, transcriptionArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, fmt.Errorf("%w", err) } return transcriptionArgs, nil default: From a7edf7b9427dc9a91f4ac60ab123b6a1be14cac9 Mon Sep 17 00:00:00 2001 From: grantdfoster Date: Thu, 23 Oct 2025 09:08:25 +0200 Subject: [PATCH 4/4] fix: error tree on unmarshall --- api/args/unmarshaller.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/args/unmarshaller.go b/api/args/unmarshaller.go index 41622540..a8a7aaa9 100644 --- a/api/args/unmarshaller.go +++ b/api/args/unmarshaller.go @@ -55,7 +55,7 @@ func UnmarshalJobArguments(jobType types.JobType, args Args) (base.JobArgument, func unmarshalWebArguments(args Args) (*web.ScraperArguments, error) { webArgs := &web.ScraperArguments{} if err := unmarshalToStruct(args, webArgs); err != nil { - return nil, fmt.Errorf("%w", err) + return nil, err } return webArgs, nil } @@ -63,25 +63,25 @@ func unmarshalWebArguments(args Args) (*web.ScraperArguments, error) { func unmarshalTikTokArguments(args Args) (base.JobArgument, error) { minimal := base.Arguments{} if err := unmarshalToStruct(args, &minimal); err != nil { - return nil, fmt.Errorf("%w", err) + return nil, err } switch minimal.Type { case types.CapSearchByQuery: searchArgs := &tiktok.QueryArguments{} if err := unmarshalToStruct(args, searchArgs); err != nil { - return nil, fmt.Errorf("%w", err) + return nil, err } return searchArgs, nil case types.CapSearchByTrending: searchArgs := &tiktok.TrendingArguments{} if err := unmarshalToStruct(args, searchArgs); err != nil { - return nil, fmt.Errorf("%w", err) + return nil, err } return searchArgs, nil case types.CapTranscription: transcriptionArgs := &tiktok.TranscriptionArguments{} if err := unmarshalToStruct(args, transcriptionArgs); err != nil { - return nil, fmt.Errorf("%w", err) + return nil, err } return transcriptionArgs, nil default: @@ -92,7 +92,7 @@ func unmarshalTikTokArguments(args Args) (base.JobArgument, error) { func unmarshalTwitterArguments(args Args) (*twitter.SearchArguments, error) { twitterArgs := &twitter.SearchArguments{} if err := unmarshalToStruct(args, twitterArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, err } return twitterArgs, nil } @@ -100,7 +100,7 @@ func unmarshalTwitterArguments(args Args) (*twitter.SearchArguments, error) { func unmarshalLinkedInArguments(args Args) (*linkedin.ProfileArguments, error) { linkedInArgs := &linkedin.ProfileArguments{} if err := unmarshalToStruct(args, linkedInArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, err } return linkedInArgs, nil } @@ -108,7 +108,7 @@ func unmarshalLinkedInArguments(args Args) (*linkedin.ProfileArguments, error) { func unmarshalRedditArguments(args Args) (*reddit.SearchArguments, error) { redditArgs := &reddit.SearchArguments{} if err := unmarshalToStruct(args, redditArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, err } return redditArgs, nil } @@ -116,7 +116,7 @@ func unmarshalRedditArguments(args Args) (*reddit.SearchArguments, error) { func unmarshalTelemetryArguments(args Args) (*telemetry.Arguments, error) { telemetryArgs := &telemetry.Arguments{} if err := unmarshalToStruct(args, telemetryArgs); err != nil { - return nil, fmt.Errorf("%w: %w", ErrFailedToUnmarshal, err) + return nil, err } return telemetryArgs, nil }