Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion DENO_README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ A [Deno](https://deno.land) client that can be used by a scraper to ease communi
],
"login_users": [
{"username": "scraping-site-username", "password": "scraping-site-password"}
]
],
// For production, set mock_mode to false
// "mock_mode": false
}
```

Expand Down
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Check file:///.../denoexample.ts
```sh
# Install latest version
go install github.com/script-development/rtcv_scraper_client/v2@latest
# Install a spesific version
# Install a specific version
go install github.com/script-development/rtcv_scraper_client/v2@v2.0.3
```

Expand All @@ -66,14 +66,18 @@ Create a `env.json` file with the following content **(this file can also be obt
],
"login_users": [
{"username": "scraping-site-username", "password": "scraping-site-password"}
]
],
// For production, set mock_mode to false
// "mock_mode": false
}
```

### *3.* Develop / Deploy a scraper using `rtcv_scraper_client`

You can now prefix your scraper's run command with `rtcv_scraper_client` and the scraper client program will run a webserver as long as your scraper runs where via you can communicate with RT-CV.

By default the program will run in `mock_mode`, for production you'll have to explicitly turn it off by setting `"mock_mode": false` in your env.json

If you have for a NodeJS project you can run your program like this:

```sh
Expand Down
2 changes: 1 addition & 1 deletion api.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (c *serverConn) DoRequest(method, path string, body, unmarshalResInto inter
if err != nil {
attempt++
if attempt > 3 {
return fmt.Errorf("%s, retried 4 times", err.Error())
return fmt.Errorf("%s retried 4 times", err.Error())
}
time.Sleep(time.Second * time.Duration(attempt) * 2)
continue
Expand Down
25 changes: 18 additions & 7 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type Env struct {
PrimaryServer EnvServer `json:"primary_server"`
AlternativeServers []EnvServer `json:"alternative_servers"`
LoginUsers []EnvUser `json:"login_users"`
MockMode bool `json:"mock_mode"`
}

func (e *Env) validate() error {
Expand Down Expand Up @@ -96,14 +97,24 @@ func main() {
credentials = append(credentials, server.toCredArg(false))
}

err = api.SetCredentials(credentials)
if err != nil {
log.Fatal(err)
// Turn on mock mode by default
api.SetMockMode()

if !env.MockMode {
err = api.SetCredentials(credentials)
if err != nil {
log.Fatal(err)
}

fmt.Println("credentials set")
fmt.Println("testing connections..")
testServerConnections(api)
fmt.Println("connected to RTCV")
} else {
fmt.Println("In mock mode")
fmt.Println("You can turn this off in by setting mock_mode to false in your env.json")
}
fmt.Println("credentials set")
fmt.Println("testing connections..")
testServerConnections(api)
fmt.Println("connected to RTCV")

fmt.Println("running scraper..")

if len(os.Args) <= 1 {
Expand Down