Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
144 commits
Select commit Hold shift + click to select a range
8e2207f
first commit
alvin-reyes May 13, 2025
965e753
add twitter search args
alvin-reyes May 14, 2025
c2b4364
add twitter search fields
alvin-reyes May 14, 2025
04223dd
add web args
alvin-reyes May 14, 2025
51a9969
add depth on web
alvin-reyes May 14, 2025
e6b98f2
add twitter result structs
alvin-reyes May 14, 2025
ba52feb
add args and types
alvin-reyes May 14, 2025
74154c9
add args and types
alvin-reyes May 14, 2025
2f2c09d
revert to use 1.23.0 go
alvin-reyes May 14, 2025
5b8af7d
remove the web types since we already have the web search args on the…
alvin-reyes May 16, 2025
2cc3690
add tiktok types
alvin-reyes Jun 6, 2025
a2cc591
Merge pull request #1 from masa-finance/feat/tiktok-types
alvin-reyes Jun 11, 2025
f54fb32
add linkedin types
alvin-reyes Jun 22, 2025
d1bd509
add linkedin
alvin-reyes Jun 22, 2025
9b46181
feat(args): extend and rename linkedin arguments for profile fetching…
teslashibe Jul 2, 2025
8573dad
docs: update README with LinkedIn profile fetching documentation
teslashibe Jul 2, 2025
1fd920c
feat: jobs type
grantdfoster Jul 23, 2025
836de78
chore: rename jobs
grantdfoster Jul 23, 2025
ab14dc7
feat: bring in sub types
grantdfoster Jul 23, 2025
32b0a29
Merge pull request #3 from masa-finance/feat--job-types
grantdfoster Jul 24, 2025
a01055a
fix: use jobtype instead of string
grantdfoster Jul 25, 2025
f321a25
Merge pull request #4 from masa-finance/feat--job-types
grantdfoster Jul 25, 2025
c2efbca
fix: remove searchbyfullarchive subtype from cred based capabilities
grantdfoster Jul 25, 2025
79e23f8
Merge pull request #5 from masa-finance/feat--job-types
grantdfoster Jul 25, 2025
9f46715
feat: refactored jobs to jobtype as key
grantdfoster Jul 28, 2025
8bb5e79
Merge pull request #6 from masa-finance/feat--job-types
grantdfoster Jul 28, 2025
51f4988
chore: adds profile type and apify caps
grantdfoster Jul 29, 2025
3bf32fc
feat: add unmarshalling for arguments per job type
grantdfoster Jul 30, 2025
38a06be
fix: better type validation
grantdfoster Jul 30, 2025
b3b2d47
fix: remove linked in from job types
grantdfoster Jul 30, 2025
1934388
chore: cleanup linkedin and twitter
grantdfoster Jul 30, 2025
e186771
fix: don't require query on twitter
grantdfoster Jul 31, 2025
a91ba05
fix: adds single tweet operation
grantdfoster Jul 31, 2025
7a10995
chore: cleanup arguments unmarshaller and job types
grantdfoster Jul 31, 2025
9d6bbeb
fix: default apify to get followers
grantdfoster Jul 31, 2025
6622c36
chore: default capability with twitter
grantdfoster Jul 31, 2025
069424e
fix: tiktok validation
grantdfoster Jul 31, 2025
df61fd0
fix: language validation
grantdfoster Jul 31, 2025
33dabcc
Add Min and Max utility functions, and the Set type
mcamou Aug 1, 2025
3d36691
Remove unneeded GH Actions workflows
mcamou Aug 1, 2025
33f7347
Fix
mcamou Aug 1, 2025
a798bba
Remove dependency on x/exp
mcamou Aug 1, 2025
382b9a6
Optimize the Set capacities a bit
mcamou Aug 1, 2025
5365f62
Union can receive multiple Sets, and NewSet returns a reference
mcamou Aug 1, 2025
08f0c8b
More stuff in .gitignore
mcamou Aug 1, 2025
0d374a8
Some more streamlining of the API
mcamou Aug 1, 2025
0e4dbd9
Merge pull request #8 from masa-finance/add-util
mcamou Aug 1, 2025
9e83399
Merge branch 'main' of github.com:masa-finance/tee-types into feat--a…
grantdfoster Aug 1, 2025
0457231
chore: captures empty capability option for job types with default fa…
grantdfoster Aug 1, 2025
d8f7dff
fix: refactor valid capability checking to job type
grantdfoster Aug 1, 2025
219c68d
chore: simplified tiktok url validation
grantdfoster Aug 1, 2025
b5867b5
fix: remove unecessary ifs
grantdfoster Aug 1, 2025
3469af4
fix: removes interface
grantdfoster Aug 1, 2025
ca00ba2
fix: favors builtin maps and go idioms for combined capabilities
grantdfoster Aug 5, 2025
208cbac
chore: lowercase error messaging
grantdfoster Aug 5, 2025
86fcdf3
update args/tiktok.go
grantdfoster Aug 7, 2025
13127c4
update args/tiktok.go
grantdfoster Aug 7, 2025
507d86b
update args/tiktok.go
grantdfoster Aug 7, 2025
a25f9c1
fix: uses utils and improves linked unmarshalling
grantdfoster Aug 7, 2025
cb4ed48
fix: adds max util where possible
grantdfoster Aug 7, 2025
d9c63c7
fix: reorder imports
grantdfoster Aug 7, 2025
7c4bb8d
fix: removes linked from available capabilities
grantdfoster Aug 7, 2025
979587f
fix: clean up linked capability
grantdfoster Aug 7, 2025
5876a14
chore: address final comments in tiktok and twitter
grantdfoster Aug 8, 2025
603d35b
Merge pull request #7 from masa-finance/feat--apify-types
grantdfoster Aug 11, 2025
b8bfa5f
fix: video url getter on tiktok tee types
grantdfoster Aug 12, 2025
482839a
fix: web unmarshaller
grantdfoster Aug 12, 2025
6df9f91
Add DO NOT MERGE / FIXME test for merging
mcamou Aug 12, 2025
e2e1f30
Add space before FIXME
mcamou Aug 12, 2025
14542ff
Merge pull request #9 from masa-finance/feat--apify-types
grantdfoster Aug 12, 2025
2df6e7b
Merge pull request #10 from masa-finance/do-not-merge
mcamou Aug 12, 2025
5d2c4db
feat: adds profile result type from scraper and json tags for twitter…
grantdfoster Aug 14, 2025
31c0685
fix: eng-US default
grantdfoster Aug 15, 2025
9a02ea5
chore: adds searchbyquery for tiktok job type
grantdfoster Aug 15, 2025
90efdd2
fix: updates default tiktok capability
grantdfoster Aug 15, 2025
5211f8e
feat: adds types for search by query on tiktok
grantdfoster Aug 15, 2025
f949598
feat: adds search by trending type
grantdfoster Aug 15, 2025
4d116a2
feat: adds searchbytrending type
grantdfoster Aug 15, 2025
ec331a7
feat: adds tiktok arg unmarshalling
grantdfoster Aug 15, 2025
b7e7dd9
feat: adds tiktok args
grantdfoster Aug 15, 2025
781daf3
fix: move tiktok caps
grantdfoster Aug 15, 2025
039c7f2
Merge pull request #12 from masa-finance/fix--tiktok-language-fix
grantdfoster Aug 18, 2025
060f7ed
Merge pull request #11 from masa-finance/feat--credential-profile-type
grantdfoster Aug 18, 2025
e9cd1f3
Merge branch 'main' of github.com:masa-finance/tee-types into feat--t…
grantdfoster Aug 21, 2025
8f9eca9
Add types for Reddit scraper
mcamou Aug 21, 2025
3a5a227
lint
mcamou Aug 21, 2025
49f78bf
test fix
mcamou Aug 21, 2025
edd7c8e
Fix Reddit args validation
mcamou Aug 21, 2025
570c8fb
Add default URL method
mcamou Aug 21, 2025
06dd40d
Lowercase query type and sort
mcamou Aug 21, 2025
3cac2f1
Some renaming of the Reddit types, and adding RedditResult
mcamou Aug 21, 2025
63523ac
Remove RedditResult (the cursor is in JobResult)
mcamou Aug 21, 2025
e480a88
PR comments
mcamou Aug 22, 2025
7fbbe94
Merge pull request #15 from masa-finance/reddit-scraper
mcamou Aug 22, 2025
52f5644
Fix Reddit unmarshal: zero values are undistinguishable from null
mcamou Aug 25, 2025
4725cd9
Move default values and canonicalization to a separate method
mcamou Aug 26, 2025
6f702b5
Merge pull request #16 from masa-finance/fix-reddit-unmarshal
mcamou Aug 26, 2025
2969314
Merge branch 'main' of github.com:masa-finance/tee-types into feat--t…
grantdfoster Aug 26, 2025
0f8ead9
fix: tiktok json labeling for search by query types
grantdfoster Aug 26, 2025
2b084a9
fix: constants
grantdfoster Aug 26, 2025
275091f
chore: uint types instead of int for search by query args
grantdfoster Aug 26, 2025
33214de
chore: force use of proxy in tiktok
grantdfoster Aug 26, 2025
0d4c4b7
chore: adds tiktok unmarshall comment and simplifies argument validation
grantdfoster Aug 26, 2025
753766d
chore: refactor args
grantdfoster Aug 26, 2025
7d6f09a
chore: simplifies unmarshaller and reorganizes telemetry args
grantdfoster Aug 26, 2025
1fdb7ee
chore: unmarshaller
grantdfoster Aug 26, 2025
b42bf7b
chore: change default tiktok capability
grantdfoster Aug 26, 2025
912f723
Merge pull request #13 from masa-finance/feat--tiktok-search
grantdfoster Aug 26, 2025
2060f38
Allow Reddit post or comment URLs only
mcamou Aug 27, 2025
c0ceec1
Lint
mcamou Aug 27, 2025
3caecf0
Merge pull request #19 from masa-finance/reddit-validation
mcamou Aug 27, 2025
bcf3377
fix: use aux instead of q
grantdfoster Aug 27, 2025
e57373c
Merge pull request #20 from masa-finance/fix--aux-over-q
grantdfoster Aug 27, 2025
f5b142f
feat: adds web and llm types and arguments
grantdfoster Sep 4, 2025
79aa76b
chore: cleanup comments
grantdfoster Sep 5, 2025
5d97ecf
fix: export LLM defaults
grantdfoster Sep 9, 2025
70e19b6
fix: export web defaults
grantdfoster Sep 9, 2025
898e896
fix: remove llm as a capability - it is not
grantdfoster Sep 9, 2025
1a8ae3d
fix: remove unused function
grantdfoster Sep 9, 2025
4de7bc7
fix: add llm response to web
grantdfoster Sep 9, 2025
3452fdc
fix: copilot suggestions
grantdfoster Sep 11, 2025
3114b62
Merge pull request #23 from masa-finance/feat--web-scraping
grantdfoster Sep 12, 2025
550b3ce
fix: add items to llm args (#25)
grantdfoster Sep 17, 2025
deaef62
fix: twitter-guard-and-job-definitions (#26)
grantdfoster Sep 23, 2025
791b003
fix: default model (#27)
grantdfoster Sep 30, 2025
9a8730b
feat: linkedin (#30)
grantdfoster Oct 9, 2025
4421902
Add collaboration rules before merge
grantdfoster Oct 13, 2025
5d3c481
Merge tee-types into tee-worker
grantdfoster Oct 13, 2025
39baf85
Complete tee-types merge
grantdfoster Oct 13, 2025
b5ee3e9
Consolidate job files and isolate tee functionality
grantdfoster Oct 13, 2025
61b37a2
Fix import issues from refactor
grantdfoster Oct 13, 2025
a8d6bd1
Fix remaining import issues in internal packages
grantdfoster Oct 13, 2025
1dd1e0f
feat: migrate tee types to worker
grantdfoster Oct 13, 2025
1e4212b
fix: duplicate gh test workflows
grantdfoster Oct 13, 2025
a5a5066
chore: cleanup reddit after refactor
grantdfoster Oct 13, 2025
dcfe3df
chore: remove .idea
grantdfoster Oct 13, 2025
18bd93f
Update pkg/util/set.go
grantdfoster Oct 13, 2025
b43ad44
fix: remove tee dependencies in client
grantdfoster Oct 14, 2025
07f299b
feat: args refactor to be consistent
grantdfoster Oct 15, 2025
5e10b98
fix: tests
grantdfoster Oct 15, 2025
248d94f
fix: linkedin test
grantdfoster Oct 15, 2025
c318467
fix: test
grantdfoster Oct 15, 2025
f78986a
fix: lint errors
grantdfoster Oct 15, 2025
31bdf65
fix: test name
grantdfoster Oct 16, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .cursor/rules/collaboration.mdc
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
---
description:
globs:
alwaysApply: true
---
# Collaboration Rules

## Planning and Confirmation Rule

**Before implementing any code changes, features, or modifications:**

1. **Create a Plan**: Always develop a clear, detailed plan that outlines:
- What changes will be made
- Which files will be modified or created
- The approach and methodology
- Expected outcomes and impacts

2. **Confirm with User**: Present the plan to the user and wait for explicit confirmation before:
- Making any file modifications
- Creating new files
- Running commands that modify the codebase
- Implementing any suggested changes

3. **Get Approval**: Only proceed with implementation after receiving clear approval from the user.

4. **No Assumptions**: Never assume the user wants changes implemented immediately, even if they seem obvious or beneficial.

**Exception**: Read-only operations (viewing files, searching, analyzing) do not require prior confirmation.

This rule ensures we maintain collaborative control over the codebase and prevents unwanted changes.
15 changes: 15 additions & 0 deletions .cursor/rules/testing.mdc
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
alwaysApply: true
---

## Testing Rule

**Before writing any test code:**

1. **Create a Test Plan**: Always develop a clear, detailed plan that outlines:
- What tests will be written
- Which files will be modified or created
- The approach and methodology
- Expected outcomes and impacts

2. **Prefer Ginkgo & Gomega**: When writing tests, use the Ginkgo and Gomega frameworks (BDD style) where possible to structure and assert tests in Go. Only use the built-in `testing` package for compatibility or legacy reasons, or if instructed.
11 changes: 11 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file

version: 2
updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ bp-todo.md
# TEE
tee/private.pem

# LLML
# LLM-related files
.aider*
GEMINI.md

# Examples from tee-types
/examples/*json
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ test-tiktok: docker-build-test
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) go test -v ./internal/jobs/tiktok_test.go ./internal/jobs/jobs_suite_test.go

test-reddit: docker-build-test
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) go test -v ./internal/jobs/reddit_test.go ./internal/jobs/redditapify/client_test.go ./api/types/reddit/reddit_suite_test.go
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) sh -c "cd /app && go test -v ./internal/jobs/reddit_test.go ./internal/jobs/jobs_suite_test.go"
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) sh -c "cd /app && go test -v ./internal/jobs/redditapify"
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) sh -c "cd /app && go test -v ./api/types/reddit_test.go"

test-web: docker-build-test
@docker run --user root $(ENV_FILE_ARG) -v $(PWD)/.masa:/home/masa -v $(PWD)/coverage:/app/coverage --rm --workdir /app -e DATA_DIR=/home/masa $(TEST_IMAGE) sh -c "cd /app && go test -v ./internal/jobs/web_test.go ./internal/jobs/jobs_suite_test.go"
Expand Down
49 changes: 49 additions & 0 deletions api/args/base/base.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package base

import (
"encoding/json"
"fmt"

"github.com/masa-finance/tee-worker/api/types"
)

// JobArgument defines the interface that all job arguments must implement
type JobArgument interface {
UnmarshalJSON([]byte) error
GetCapability() types.Capability
ValidateCapability(jobType types.JobType) error
SetDefaultValues()
Validate() error
}

// Verify interface implementation
var _ JobArgument = (*Arguments)(nil)

type Arguments struct {
Type types.Capability `json:"type"`
}

func (t *Arguments) UnmarshalJSON(data []byte) error {
type Alias Arguments
aux := &struct{ *Alias }{Alias: (*Alias)(t)}
if err := json.Unmarshal(data, aux); err != nil {
return fmt.Errorf("%v: %w", "failed to unmarshal arguments", err)
}
t.SetDefaultValues()
return t.Validate()
}

func (a *Arguments) GetCapability() types.Capability {
return a.Type
}

func (a *Arguments) ValidateCapability(jobType types.JobType) error {
return jobType.ValidateCapability(&a.Type)
}

func (a *Arguments) SetDefaultValues() {
}

func (a *Arguments) Validate() error {
return nil
}
7 changes: 7 additions & 0 deletions api/args/linkedin/linkedin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package linkedin

import (
"github.com/masa-finance/tee-worker/api/args/linkedin/profile"
)

type Profile = profile.Arguments
141 changes: 141 additions & 0 deletions api/args/linkedin/profile/profile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package profile

import (
"encoding/json"
"errors"
"fmt"

"github.com/masa-finance/tee-worker/api/args/base"
"github.com/masa-finance/tee-worker/api/types"
"github.com/masa-finance/tee-worker/api/types/linkedin/experiences"
"github.com/masa-finance/tee-worker/api/types/linkedin/functions"
"github.com/masa-finance/tee-worker/api/types/linkedin/industries"
"github.com/masa-finance/tee-worker/api/types/linkedin/profile"
"github.com/masa-finance/tee-worker/api/types/linkedin/seniorities"
)

var (
ErrScraperModeNotSupported = errors.New("scraper mode not supported")
ErrMaxItemsTooLarge = errors.New("max items must be less than or equal to 100")
ErrExperienceNotSupported = errors.New("years of experience not supported")
ErrSeniorityNotSupported = errors.New("seniority level not supported")
ErrFunctionNotSupported = errors.New("function not supported")
ErrIndustryNotSupported = errors.New("industry not supported")
ErrUnmarshalling = errors.New("failed to unmarshal LinkedIn profile arguments")
)

const (
DefaultMaxItems = 10
DefaultScraperMode = profile.ScraperModeShort
MaxItems = 1000 // 2500 on the actor, but we will run over 1MB memory limit on responses
)

// Verify interface implementation
var _ base.JobArgument = (*Arguments)(nil)

// Arguments defines args for LinkedIn profile operations
type Arguments struct {
Type types.Capability `json:"type"`
ScraperMode profile.ScraperMode `json:"profileScraperMode"`
Query string `json:"searchQuery"`
MaxItems uint `json:"maxItems"`
Locations []string `json:"locations,omitempty"`
CurrentCompanies []string `json:"currentCompanies,omitempty"`
PastCompanies []string `json:"pastCompanies,omitempty"`
CurrentJobTitles []string `json:"currentJobTitles,omitempty"`
PastJobTitles []string `json:"pastJobTitles,omitempty"`
Schools []string `json:"schools,omitempty"`
YearsOfExperience []experiences.Id `json:"yearsOfExperienceIds,omitempty"`
YearsAtCurrentCompany []experiences.Id `json:"yearsAtCurrentCompanyIds,omitempty"`
SeniorityLevels []seniorities.Id `json:"seniorityLevelIds,omitempty"`
Functions []functions.Id `json:"functionIds,omitempty"`
Industries []industries.Id `json:"industryIds,omitempty"`
FirstNames []string `json:"firstNames,omitempty"`
LastNames []string `json:"lastNames,omitempty"`
RecentlyChangedJobs bool `json:"recentlyChangedJobs,omitempty"`
StartPage uint `json:"startPage,omitempty"`
}

func (t *Arguments) UnmarshalJSON(data []byte) error {
type Alias Arguments
aux := &struct{ *Alias }{Alias: (*Alias)(t)}
if err := json.Unmarshal(data, aux); err != nil {
return fmt.Errorf("%w: %w", ErrUnmarshalling, err)
}
t.SetDefaultValues()
return t.Validate()
}

func (a *Arguments) SetDefaultValues() {
if a.MaxItems == 0 {
a.MaxItems = DefaultMaxItems
}
if a.ScraperMode == "" {
a.ScraperMode = DefaultScraperMode
}
}

func (a *Arguments) Validate() error {
var errs []error

if a.MaxItems > MaxItems {
errs = append(errs, ErrMaxItemsTooLarge)
}

err := a.ValidateCapability(types.LinkedInJob)
if err != nil {
errs = append(errs, err)
}

if !profile.AllScraperModes.Contains(a.ScraperMode) {
errs = append(errs, ErrScraperModeNotSupported)
}

for _, yoe := range a.YearsOfExperience {
if !experiences.All.Contains(yoe) {
errs = append(errs, fmt.Errorf("%w: %v", ErrExperienceNotSupported, yoe))
}
}
for _, yac := range a.YearsAtCurrentCompany {
if !experiences.All.Contains(yac) {
errs = append(errs, fmt.Errorf("%w: %v", ErrExperienceNotSupported, yac))
}
}
for _, sl := range a.SeniorityLevels {
if !seniorities.All.Contains(sl) {
errs = append(errs, fmt.Errorf("%w: %v", ErrSeniorityNotSupported, sl))
}
}
for _, f := range a.Functions {
if !functions.All.Contains(f) {
errs = append(errs, fmt.Errorf("%w: %v", ErrFunctionNotSupported, f))
}
}
for _, i := range a.Industries {
if !industries.All.Contains(i) {
errs = append(errs, fmt.Errorf("%w: %v", ErrIndustryNotSupported, i))
}
}

if len(errs) > 0 {
return errors.Join(errs...)
}

return nil
}

func (a *Arguments) GetCapability() types.Capability {
return a.Type
}

func (a *Arguments) ValidateCapability(jobType types.JobType) error {
return jobType.ValidateCapability(&a.Type)
}

// NewArguments creates a new Arguments instance and applies default values immediately
func NewArguments() Arguments {
args := Arguments{}
args.SetDefaultValues()
args.Validate() // This will set the default capability via ValidateCapability
return args
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package reddit_test
package profile_test

import (
"testing"
Expand All @@ -7,7 +7,7 @@ import (
. "github.com/onsi/gomega"
)

func TestReddit(t *testing.T) {
func TestArgs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Reddit Suite")
}
RunSpecs(t, "Args Suite")
}
Loading
Loading