From 2dc5b4d99e0a376ebeb3d35f975c5ca7b7c69b2c Mon Sep 17 00:00:00 2001 From: cmj Date: Wed, 12 Jul 2023 15:42:14 +0800 Subject: [PATCH 1/5] feat(demo): provides a demo sub-command --- clotho.go | 7 +++ internal/demo/command.go | 105 +++++++++++++++++++++++++++++++++++ internal/linkedin/command.go | 17 +++--- internal/utils/selenium.go | 2 +- 4 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 internal/demo/command.go diff --git a/clotho.go b/clotho.go index 98f415e..3e4cf4a 100644 --- a/clotho.go +++ b/clotho.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" + "github.com/cmj0121/clotho/internal/demo" "github.com/cmj0121/clotho/internal/github" "github.com/cmj0121/clotho/internal/linkedin" @@ -27,6 +28,7 @@ type Clotho struct { Github *github.GitHub `cmd:"" help:"The GitHub user collector."` LinkedIn *linkedin.LinkedIn `cmd:"" name:"linkedin" help:"The LinkedIn user collector."` + Demo *demo.Demo `cmd:"" name:"demo" help:"The Demo collector."` } // Create the new instance of the Clotho. @@ -50,6 +52,8 @@ func (c *Clotho) Run() (exitcode int) { command = c.Github case "linkedin ": command = c.LinkedIn + case "demo ": + command = c.Demo default: log.Error().Str("subcmd", sub).Msg("Sub-command not implemented.") exitcode = 1 @@ -95,6 +99,9 @@ func (c *Clotho) run(cmd SubCommand) (exitcode int) { // table.SetAutoMergeCells(true) switch resp.(type) { + case nil: + // empty result and nothing to do + return case [][]string: table.AppendBulk(resp.([][]string)) case map[string]interface{}: diff --git a/internal/demo/command.go b/internal/demo/command.go new file mode 100644 index 0000000..9744c87 --- /dev/null +++ b/internal/demo/command.go @@ -0,0 +1,105 @@ +// Get the Demo CLI interface. +package demo + +import ( + "strings" + "bufio" + "io" + "os" + "fmt" + "errors" + "net/http" + + "github.com/cmj0121/clotho/internal/utils" + "github.com/rs/zerolog/log" +) + +type Demo struct { + Link string `arg:"" help:"The target link."` + Action string `enum:"http,selenium" required:"" default:"http" help:"The action to perform."` + + // The HTTP client for the GitHub API. + utils.Client + // The Selenium wrapper client. + utils.Selenium +} + +func (d *Demo) Prologue() { + switch d.Action { + case "http": + d.Client.Prologue() + case "selenium": + d.Selenium.Prologue() + } +} + +func (d *Demo) Epilogue() { + switch d.Action { + case "http": + d.Client.Epilogue() + case "selenium": + d.Selenium.Epilogue() + } +} + +// Get the GitHub user information. +func (d *Demo) Execute() (data interface{}, err error) { + log.Info().Str("action", d.Action).Msg("execute the demo command") + + switch d.Action { + case "http": + var resp *http.Response + + resp, err = d.Client.Get(d.Link) + if err != nil { + log.Error().Err(err).Msg("failed to get the link") + return + } + defer resp.Body.Close() + + var body []byte + body, err = io.ReadAll(resp.Body) + if err != nil { + log.Error().Err(err).Msg("failed to read the response body") + return + } + + fmt.Println(string(body)) + case "selenium": + help := strings.Join([]string{ + "exit: exit the demo command", + }, "\n") + + reader := bufio.NewReader(os.Stdin) + d.Selenium.Get(d.Link) + for { + fmt.Print(">>> ") + + text, rerr := reader.ReadString('\n') + switch rerr { + case nil: + text = strings.TrimSpace(text) + switch text { + case "exit": + return + default: + fmt.Printf("unknown command: %v\n", text) + fmt.Println(help) + } + case io.EOF: + return + default: + log.Error().Err(rerr).Msg("failed to read the command") + + err = errors.New(fmt.Sprintf("failed to read the command: %v", rerr)) + return + } + } + + default: + err = errors.New(fmt.Sprintf("unknown action: %v", d.Action)) + return + } + + return +} diff --git a/internal/linkedin/command.go b/internal/linkedin/command.go index 308c1f3..c11be8b 100644 --- a/internal/linkedin/command.go +++ b/internal/linkedin/command.go @@ -3,17 +3,17 @@ package linkedin import ( "fmt" - "time" "strings" + "time" "github.com/cmj0121/clotho/internal/utils" - "github.com/tebeka/selenium" "github.com/rs/zerolog/log" + "github.com/tebeka/selenium" ) type LinkedIn struct { - Username string `arg:"" help:"The GitHub username."` - Wait time.Duration `help:"The wait time for the Selenium dom visible." default:"2s"` + Username string `arg:"" help:"The GitHub username."` + Wait time.Duration `help:"The wait time for the Selenium dom visible." default:"2s"` // The Selenium wrapper client. utils.Selenium @@ -33,9 +33,9 @@ func (l *LinkedIn) Execute() (resp interface{}, err error) { func (l *LinkedIn) extractUserProfile() (data [][]string) { data = [][]string{ - []string{"name", l.getFieldData(".top-card-layout__title"), "", ""}, - []string{"title", l.getFieldData(".top-card-layout__headline"), "", ""}, - []string{"location", l.getFieldData(".top-card__subline-item"), "", ""}, + {"name", l.getFieldData(".top-card-layout__title"), "", ""}, + {"title", l.getFieldData(".top-card-layout__headline"), "", ""}, + {"location", l.getFieldData(".top-card__subline-item"), "", ""}, } // wait untils the experience section is loaded @@ -78,7 +78,6 @@ func (l *LinkedIn) extractUserProfile() (data [][]string) { for _, position := range positions { title, _ := l.getText(position, "h3.profile-section-card__title") - time_ranges_dom, _ := position.FindElements(selenium.ByCSSSelector, ".date-range > time") time_ranges := []string{} for _, time_range_dom := range time_ranges_dom { @@ -114,7 +113,7 @@ func (l *LinkedIn) getFieldData(selector string) (value string) { return } -func (l *LinkedIn) getText(elm selenium.WebElement, selectors... string) (value string, err error) { +func (l *LinkedIn) getText(elm selenium.WebElement, selectors ...string) (value string, err error) { var dom selenium.WebElement for _, selector := range selectors { diff --git a/internal/utils/selenium.go b/internal/utils/selenium.go index 871c101..c8fd71e 100644 --- a/internal/utils/selenium.go +++ b/internal/utils/selenium.go @@ -22,7 +22,7 @@ type Selenium struct { Driver string `group:"selenium" help:"The path to the ChromeDriver." default:"chromedriver"` Port int `group:"selenium" help:"The bind address for the ChromeDriver." default:"9515"` DriverVersion string `group:"selenium" help:"The version of the ChromeDriver." default:""` - Headless bool `group:"selenium" help:"Run the ChromeDriver in headless mode." default:"true" negatable:""` + Headless bool `group:"selenium" help:"Run the ChromeDriver in headless mode." default:"true" negatable:""` // The Selenium service for the ChromeDriver. *selenium.Service `kong:"-"` From 55a11a5ac4a571a0d402bf60d1de62693fb27863 Mon Sep 17 00:00:00 2001 From: cmj Date: Fri, 14 Jul 2023 16:51:57 +0800 Subject: [PATCH 2/5] feat(utils): provides the chromedp as the another solution for browser chromedp/chromedp is more activate than tebeka/selenium --- Makefile | 3 ++- go.mod | 28 ++++++++++++++++---- go.sum | 53 ++++++++++++++++++++++++++++++++++--- internal/demo/command.go | 36 +++++++++++++------------- internal/utils/chrome.go | 56 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 internal/utils/chrome.go diff --git a/Makefile b/Makefile index 01a26fe..836dcbf 100644 --- a/Makefile +++ b/Makefile @@ -35,5 +35,6 @@ help: # show this message awk 'BEGIN {FS = ":.*?#"} {printf " %-18s %s\n", $$1, $$2}' dist/%: cmd/%/main.go $(SRC) - mkdir -p $(@D) + @mkdir -p $(@D) + @go mod tidy go build -ldflags="-s -w" -o $@ $< diff --git a/go.mod b/go.mod index 6d09a99..6d6378d 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,31 @@ module github.com/cmj0121/clotho go 1.20 require ( - github.com/alecthomas/kong v0.8.0 // indirect + github.com/alecthomas/kong v0.8.0 + github.com/chromedp/chromedp v0.9.1 + github.com/go-rod/stealth v0.4.9 + github.com/olekukonko/tablewriter v0.0.5 + github.com/rs/zerolog v1.29.1 + github.com/tebeka/selenium v0.9.9 +) + +require ( github.com/blang/semver v3.5.1+incompatible // indirect + github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 // indirect + github.com/chromedp/sysutil v1.0.0 // indirect + github.com/go-rod/rod v0.113.0 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/rs/zerolog v1.29.1 // indirect - github.com/tebeka/selenium v0.9.9 // indirect - golang.org/x/sys v0.10.0 // indirect + github.com/ysmood/fetchup v0.2.3 // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/got v0.34.1 // indirect + github.com/ysmood/gson v0.7.3 // indirect + github.com/ysmood/leakless v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index 32b1bf9..21c042a 100644 --- a/go.sum +++ b/go.sum @@ -3,16 +3,38 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA= github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k= +github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s= github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY= +github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= +github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ= +github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms= +github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4= +github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -22,6 +44,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -32,7 +55,14 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -41,12 +71,28 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w= github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc= +github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= +github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/gop v0.0.2 h1:VuWweTmXK+zedLqYufJdh3PlxDNBOfFHjIZlPT2T5nw= +github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= +github.com/ysmood/got v0.34.1 h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s= +github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM= +github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= +github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= +github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -69,6 +115,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -86,11 +133,11 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/internal/demo/command.go b/internal/demo/command.go index 9744c87..fcd67e5 100644 --- a/internal/demo/command.go +++ b/internal/demo/command.go @@ -2,13 +2,13 @@ package demo import ( - "strings" "bufio" - "io" - "os" - "fmt" "errors" + "fmt" + "io" "net/http" + "os" + "strings" "github.com/cmj0121/clotho/internal/utils" "github.com/rs/zerolog/log" @@ -77,22 +77,22 @@ func (d *Demo) Execute() (data interface{}, err error) { text, rerr := reader.ReadString('\n') switch rerr { - case nil: - text = strings.TrimSpace(text) - switch text { - case "exit": - return - default: - fmt.Printf("unknown command: %v\n", text) - fmt.Println(help) - } - case io.EOF: + case nil: + text = strings.TrimSpace(text) + switch text { + case "exit": return default: - log.Error().Err(rerr).Msg("failed to read the command") - - err = errors.New(fmt.Sprintf("failed to read the command: %v", rerr)) - return + fmt.Printf("unknown command: %v\n", text) + fmt.Println(help) + } + case io.EOF: + return + default: + log.Error().Err(rerr).Msg("failed to read the command") + + err = errors.New(fmt.Sprintf("failed to read the command: %v", rerr)) + return } } diff --git a/internal/utils/chrome.go b/internal/utils/chrome.go new file mode 100644 index 0000000..51194ff --- /dev/null +++ b/internal/utils/chrome.go @@ -0,0 +1,56 @@ +// Get the global utility for the internal packages. +package utils + +import ( + "context" + + "github.com/chromedp/chromedp" + "github.com/go-rod/stealth" +) + +// The chrome wrapper with undetectable techniques. +type Chrome struct { + // The ChromeDP parent context. + exec_ctx context.Context + exec_cancel context.CancelFunc + + // The ChromeDP child context. + ctx context.Context + cancel context.CancelFunc +} + +// open the necessary resources. +func (c *Chrome) Prologue() { + // The customized chrome options. + opts := append(chromedp.DefaultExecAllocatorOptions[:], + // general chrome options + chromedp.DisableGPU, + chromedp.NoFirstRun, + chromedp.NoDefaultBrowserCheck, + // make the browser window undetecteable for webdriver detection. + chromedp.Flag("enable-automation", false), + chromedp.Flag("disable-blink-features", "AutomationControlled"), + // headless chrome options + chromedp.Flag("headless", false), + ) + + c.exec_ctx, c.exec_cancel = chromedp.NewExecAllocator(context.Background(), opts...) + c.ctx, c.cancel = chromedp.NewContext(c.exec_ctx) +} + +// clean up the resources. +func (c *Chrome) Epilogue() { + c.cancel() + c.exec_cancel() +} + +// Navigate to the given url. +func (c *Chrome) Navigate(url string) (err error) { + err = chromedp.Run( + c.ctx, + // insert the stealth-js to the browser before the page is loaded + chromedp.Evaluate(stealth.JS, nil), + chromedp.Navigate(url), + ) + return +} From 17ee09ecb6b13742c2c6452f3e04f9c43c0be875 Mon Sep 17 00:00:00 2001 From: cmj Date: Fri, 14 Jul 2023 16:59:52 +0800 Subject: [PATCH 3/5] feat(demo): remove the old selenium solution Use the chromedp as the default chrome solution --- go.mod | 4 +- go.sum | 114 +------------------- internal/demo/command.go | 36 +++---- internal/utils/chrome.go | 4 +- internal/utils/selenium.go | 213 ------------------------------------- 5 files changed, 23 insertions(+), 348 deletions(-) delete mode 100644 internal/utils/selenium.go diff --git a/go.mod b/go.mod index 6d6378d..358e76d 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,9 @@ require ( github.com/go-rod/stealth v0.4.9 github.com/olekukonko/tablewriter v0.0.5 github.com/rs/zerolog v1.29.1 - github.com/tebeka/selenium v0.9.9 ) require ( - github.com/blang/semver v3.5.1+incompatible // indirect github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/go-rod/rod v0.113.0 // indirect @@ -29,5 +27,5 @@ require ( github.com/ysmood/got v0.34.1 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 21c042a..851a4d2 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,13 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA= -github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k= github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s= github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY= github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms= github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= @@ -34,31 +20,9 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -77,8 +41,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= -github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w= -github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc= github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= @@ -93,82 +55,10 @@ github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/demo/command.go b/internal/demo/command.go index fcd67e5..410a4b2 100644 --- a/internal/demo/command.go +++ b/internal/demo/command.go @@ -16,20 +16,20 @@ import ( type Demo struct { Link string `arg:"" help:"The target link."` - Action string `enum:"http,selenium" required:"" default:"http" help:"The action to perform."` + Action string `enum:"http,chrome" required:"" default:"http" help:"The action to perform."` // The HTTP client for the GitHub API. utils.Client - // The Selenium wrapper client. - utils.Selenium + // The Chrome wrapper client. + utils.Chrome } func (d *Demo) Prologue() { switch d.Action { case "http": d.Client.Prologue() - case "selenium": - d.Selenium.Prologue() + case "chrome": + d.Chrome.Prologue() } } @@ -37,8 +37,8 @@ func (d *Demo) Epilogue() { switch d.Action { case "http": d.Client.Epilogue() - case "selenium": - d.Selenium.Epilogue() + case "chrome": + d.Chrome.Epilogue() } } @@ -65,26 +65,24 @@ func (d *Demo) Execute() (data interface{}, err error) { } fmt.Println(string(body)) - case "selenium": - help := strings.Join([]string{ - "exit: exit the demo command", - }, "\n") - + case "chrome": reader := bufio.NewReader(os.Stdin) - d.Selenium.Get(d.Link) + link := d.Link for { - fmt.Print(">>> ") + d.Chrome.Navigate(link) + fmt.Print(">>> ") text, rerr := reader.ReadString('\n') + switch rerr { case nil: text = strings.TrimSpace(text) - switch text { - case "exit": - return + + switch { + case strings.HasPrefix(text, "http://") || strings.HasPrefix(text, "https://"): + link = text default: - fmt.Printf("unknown command: %v\n", text) - fmt.Println(help) + fmt.Printf("invalid link: %s\n", text) } case io.EOF: return diff --git a/internal/utils/chrome.go b/internal/utils/chrome.go index 51194ff..9632d0e 100644 --- a/internal/utils/chrome.go +++ b/internal/utils/chrome.go @@ -10,6 +10,8 @@ import ( // The chrome wrapper with undetectable techniques. type Chrome struct { + Headless bool `group:"chromr" help:"Run the browser in headless mode." default:"true" negatable:""` + // The ChromeDP parent context. exec_ctx context.Context exec_cancel context.CancelFunc @@ -31,7 +33,7 @@ func (c *Chrome) Prologue() { chromedp.Flag("enable-automation", false), chromedp.Flag("disable-blink-features", "AutomationControlled"), // headless chrome options - chromedp.Flag("headless", false), + chromedp.Flag("headless", c.Headless), ) c.exec_ctx, c.exec_cancel = chromedp.NewExecAllocator(context.Background(), opts...) diff --git a/internal/utils/selenium.go b/internal/utils/selenium.go deleted file mode 100644 index c8fd71e..0000000 --- a/internal/utils/selenium.go +++ /dev/null @@ -1,213 +0,0 @@ -// Get the global utility for the internal packages. -package utils - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "path" - "regexp" - "runtime" - "time" - - "github.com/rs/zerolog/log" - "github.com/tebeka/selenium" - "github.com/tebeka/selenium/chrome" -) - -type Selenium struct { - // The options for the Selenium client. - Driver string `group:"selenium" help:"The path to the ChromeDriver." default:"chromedriver"` - Port int `group:"selenium" help:"The bind address for the ChromeDriver." default:"9515"` - DriverVersion string `group:"selenium" help:"The version of the ChromeDriver." default:""` - Headless bool `group:"selenium" help:"Run the ChromeDriver in headless mode." default:"true" negatable:""` - - // The Selenium service for the ChromeDriver. - *selenium.Service `kong:"-"` - - // The Selenium web driver for the ChromeDriver. - selenium.WebDriver `kong:"-"` - - // The internal HTTP client. - client Client `kong:"-"` -} - -// open the necessary resources for the HTTP client. -func (s *Selenium) Prologue() { - s.client.Prologue() - - if err := s.setupDriver(); err != nil { - log.Error().Err(err).Msg("failed to setup the ChromeDriver") - panic(err) - } - - opts := []selenium.ServiceOption{} - - service, err := selenium.NewChromeDriverService(s.Driver, s.Port, opts...) - if err != nil { - log.Error().Err(err).Msg("failed to start the ChromeDriver service") - panic(err) - } - - s.Service = service - - caps := selenium.Capabilities{"browserName": "chrome"} - if s.Headless { - log.Info().Msg("run the ChromeDriver in headless mode") - - caps.AddChrome(chrome.Capabilities{ - Args: []string{ - "--headless", - "--disable-gpu", - "--no-sandbox", - "--disable-dev-shm-usage", - }, - }) - } - - wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", s.Port)) - if err != nil { - log.Error().Err(err).Msg("failed to start the ChromeDriver") - panic(err) - } - - s.WebDriver = wd -} - -// clean up the resources for the HTTP client. -func (s *Selenium) Epilogue() { - s.client.Epilogue() - - if s.WebDriver != nil { - s.WebDriver.Quit() - s.WebDriver = nil - } - - if s.Service != nil { - s.Service.Stop() - s.Service = nil - } -} - -// wait until the element is visible. -func (s *Selenium) WaitVisible(selector string, duration time.Duration) (exists bool) { - // wait until the element is visible. - s.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) { - _, err := wd.FindElement(selenium.ByCSSSelector, selector) - return err == nil, nil - }, duration) - - _, err := s.FindElement(selenium.ByCSSSelector, selector) - return err == nil -} - -// check the driver exists or not, and download it if necessary. -func (s *Selenium) setupDriver() (err error) { - if _, err = os.Stat(s.Driver); os.IsNotExist(err) { - if err = s.downloadDriver(); err != nil { - log.Error().Err(err).Msg("failed to download the ChromeDriver") - return - } - } - - return -} - -// get the latest driver version from the official site. -func (s *Selenium) getLatestDriverVersion() (ver string, err error) { - log.Debug().Str("driver", s.Driver).Msg("downloading the ChromeDriver") - - // download the driver from the official site - // - https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${MAJOR} - // - https://chromedriver.storage.googleapis.com/${VERSION}/chromedriver_linux64.zip - - // get the latest chromedriver version from CLI option - url := "https://chromedriver.storage.googleapis.com/LATEST_RELEASE" - if s.DriverVersion != "" { - log.Info().Str("version", s.DriverVersion).Msg("use the specific version") - url = fmt.Sprintf("%v_%v", url, s.DriverVersion) - } - - var resp *http.Response - - resp, err = s.client.Get(url) - if err != nil { - log.Warn().Err(err).Str("url", url).Msg("failed to get the latest version") - return - } - - defer resp.Body.Close() - data, _ := io.ReadAll(resp.Body) - - // it should be XXXX.XXXX.XXXX.XXXX - ver = string(data) - if match, _ := regexp.MatchString(`^\d+\.\d+\.\d+\.\d+$`, ver); !match { - log.Warn().Str("version", s.DriverVersion).Msg("invalid driver version") - return - } - - return -} - -// download the driver from the official site. -func (s *Selenium) downloadDriver() (err error) { - var latest_ver string - - latest_ver, err = s.getLatestDriverVersion() - if err != nil { - log.Error().Err(err).Msg("failed to get the latest version") - return - } - - var platform string - - switch runtime.GOOS { - case "linux": - platform = "linux64" - case "darwin": - if platform = "mac64"; runtime.GOARCH != "amd64" { - log.Info().Str("arch", runtime.GOARCH).Msg("use the arm64 version") - platform = "mac_arm64" - } - default: - log.Warn().Str("os", runtime.GOOS).Msg("unsupported platform") - err = errors.New("unsupported platform") - return - } - - url := fmt.Sprintf("https://chromedriver.storage.googleapis.com/%v/chromedriver_%v.zip", latest_ver, platform) - log.Debug().Str("url", url).Msg("downloading the ChromeDriver") - - var resp *http.Response - resp, err = s.client.Get(url) - if err != nil { - log.Error().Err(err).Str("url", url).Msg("failed to download the ChromeDriver") - return - } - - defer resp.Body.Close() - - dir := path.Dir(s.Driver) - if err = os.MkdirAll(dir, 0755); err != nil { - log.Error().Err(err).Str("dir", dir).Msg("failed to create the directory") - return - } - - var file *os.File - file, err = os.OpenFile(s.Driver, os.O_CREATE|os.O_WRONLY, 0755) - if err != nil { - log.Error().Err(err).Str("file", s.Driver).Msg("failed to create the file") - return - } - - defer file.Close() - - if _, err = io.Copy(file, resp.Body); err != nil { - log.Error().Err(err).Str("file", s.Driver).Msg("failed to write the file") - return - } - - return -} From de5a7e7cc916c899f513826b332818a4ec2cb605 Mon Sep 17 00:00:00 2001 From: cmj Date: Fri, 14 Jul 2023 17:12:43 +0800 Subject: [PATCH 4/5] fixup! feat(demo): remove the old selenium solution --- internal/utils/chrome.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/utils/chrome.go b/internal/utils/chrome.go index 9632d0e..cc687f9 100644 --- a/internal/utils/chrome.go +++ b/internal/utils/chrome.go @@ -10,7 +10,7 @@ import ( // The chrome wrapper with undetectable techniques. type Chrome struct { - Headless bool `group:"chromr" help:"Run the browser in headless mode." default:"true" negatable:""` + Headless bool `group:"chrome" help:"Run the browser in headless mode." default:"true" negatable:""` // The ChromeDP parent context. exec_ctx context.Context From 12792035463fa651de4b0c4472e05735711d589a Mon Sep 17 00:00:00 2001 From: cmj Date: Fri, 14 Jul 2023 17:14:15 +0800 Subject: [PATCH 5/5] feat(linkedin): remove the LinkedIn collector --- clotho.go | 4 - go.sum | 1 - internal/linkedin/command.go | 137 ----------------------------------- internal/utils/chrome.go | 6 ++ 4 files changed, 6 insertions(+), 142 deletions(-) delete mode 100644 internal/linkedin/command.go diff --git a/clotho.go b/clotho.go index 3e4cf4a..ce1548c 100644 --- a/clotho.go +++ b/clotho.go @@ -7,7 +7,6 @@ import ( "github.com/cmj0121/clotho/internal/demo" "github.com/cmj0121/clotho/internal/github" - "github.com/cmj0121/clotho/internal/linkedin" "github.com/alecthomas/kong" "github.com/olekukonko/tablewriter" @@ -27,7 +26,6 @@ type Clotho struct { Verbose int `short:"v" group:"logger" xor:"verbose,quiet" type:"counter" help:"Show the verbose logger."` Github *github.GitHub `cmd:"" help:"The GitHub user collector."` - LinkedIn *linkedin.LinkedIn `cmd:"" name:"linkedin" help:"The LinkedIn user collector."` Demo *demo.Demo `cmd:"" name:"demo" help:"The Demo collector."` } @@ -50,8 +48,6 @@ func (c *Clotho) Run() (exitcode int) { switch sub := ctx.Command(); sub { case "github ": command = c.Github - case "linkedin ": - command = c.LinkedIn case "demo ": command = c.Demo default: diff --git a/go.sum b/go.sum index 851a4d2..6e19257 100644 --- a/go.sum +++ b/go.sum @@ -58,7 +58,6 @@ github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/linkedin/command.go b/internal/linkedin/command.go deleted file mode 100644 index c11be8b..0000000 --- a/internal/linkedin/command.go +++ /dev/null @@ -1,137 +0,0 @@ -// Get the LinkedIn CLI interface. -package linkedin - -import ( - "fmt" - "strings" - "time" - - "github.com/cmj0121/clotho/internal/utils" - "github.com/rs/zerolog/log" - "github.com/tebeka/selenium" -) - -type LinkedIn struct { - Username string `arg:"" help:"The GitHub username."` - Wait time.Duration `help:"The wait time for the Selenium dom visible." default:"2s"` - - // The Selenium wrapper client. - utils.Selenium -} - -func (l *LinkedIn) Execute() (resp interface{}, err error) { - url := fmt.Sprintf("https://www.linkedin.com/in/%s", l.Username) - - if err = l.Get(url); err != nil { - log.Error().Str("url", url).Err(err).Msg("failed to get the LinkedIn profile") - return - } - - resp = l.extractUserProfile() - return -} - -func (l *LinkedIn) extractUserProfile() (data [][]string) { - data = [][]string{ - {"name", l.getFieldData(".top-card-layout__title"), "", ""}, - {"title", l.getFieldData(".top-card-layout__headline"), "", ""}, - {"location", l.getFieldData(".top-card__subline-item"), "", ""}, - } - - // wait untils the experience section is loaded - selector := ".experience__list > li" - exists := l.WaitVisible(selector, l.Wait) - if !exists { - log.Warn().Str("wait", l.Wait.String()).Msg("failed to find the experience section") - return - } - - // get the experience section - elms, _ := l.FindElements(selenium.ByCSSSelector, selector) - for _, elm := range elms { - class, err := elm.GetAttribute("class") - if err != nil { - log.Warn().Err(err).Msg("failed to get the class attribute") - continue - } - - if strings.Contains(class, "profile-section-card") { - company, _ := l.getText(elm, ".profile-section-card__subtitle > a", "h4.profile-section-card__subtitle") - title, _ := l.getText(elm, "h3.profile-section-card__title") - - time_ranges_dom, _ := elm.FindElements(selenium.ByCSSSelector, ".date-range > time") - time_ranges := []string{} - for _, time_range_dom := range time_ranges_dom { - time_range, _ := time_range_dom.GetAttribute("innerText") - time_ranges = append(time_ranges, time_range) - } - - data = append(data, []string{"", company, title, strings.Join(time_ranges, " - ")}) - continue - } - - if strings.Contains(class, "experience-group") { - company, _ := l.getText(elm, "h4.experience-group-header__company") - - selector := ".experience-group-position" - positions, _ := l.FindElements(selenium.ByCSSSelector, selector) - for _, position := range positions { - title, _ := l.getText(position, "h3.profile-section-card__title") - - time_ranges_dom, _ := position.FindElements(selenium.ByCSSSelector, ".date-range > time") - time_ranges := []string{} - for _, time_range_dom := range time_ranges_dom { - time_range, _ := time_range_dom.GetAttribute("innerText") - time_ranges = append(time_ranges, time_range) - } - - data = append(data, []string{"", company, title, strings.Join(time_ranges, " - ")}) - } - - continue - } - - log.Info().Str("class", class).Msg("unknown element") - } - - return -} - -func (l *LinkedIn) getFieldData(selector string) (value string) { - element, err := l.FindElement(selenium.ByCSSSelector, selector) - if err != nil { - log.Info().Str("selector", selector).Err(err).Msg("failed to find the element") - return - } - - value, err = element.Text() - if err != nil { - log.Error().Str("selector", selector).Err(err).Msg("failed to get the text of the element") - return - } - - return -} - -func (l *LinkedIn) getText(elm selenium.WebElement, selectors ...string) (value string, err error) { - var dom selenium.WebElement - - for _, selector := range selectors { - dom, err = elm.FindElement(selenium.ByCSSSelector, selector) - if err != nil { - log.Debug().Str("selector", selector).Err(err).Msg("failed to find the element") - continue - } - - value, err = dom.GetAttribute("innerText") - if err != nil { - log.Error().Err(err).Msg("failed to get the text of the element") - continue - } - - err = nil - break - } - - return -} diff --git a/internal/utils/chrome.go b/internal/utils/chrome.go index cc687f9..0449003 100644 --- a/internal/utils/chrome.go +++ b/internal/utils/chrome.go @@ -56,3 +56,9 @@ func (c *Chrome) Navigate(url string) (err error) { ) return } + +// Run the sevearal actions. +func (c *Chrome) Run(actions ...chromedp.Action) (err error) { + err = chromedp.Run(c.ctx, actions...) + return +}