-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add ticker and scheduler to pool package Add TestAndSet to rmap package Fix bug in pool package that logged incorrect errors on stopping jobs * Add scheduler pool example. Fix issue with job dispatch return status handling. * Document TestAndSet
- Loading branch information
Showing
19 changed files
with
1,128 additions
and
173 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
# pool Example | ||
|
||
This example shows how to use the pool package to create a pool of workers. It has three parts: | ||
|
||
1. The `worker` process registers a worker with the node and waits for jobs. | ||
2. The `producer` process starts and stops two jobs. It also notifies the worker handling the second job. | ||
3. The `scheduler` process starts runs a schedule that starts and stops jobs alternately. | ||
|
||
## Running the example | ||
|
||
To run the example, first make sure Redis is running locally. The `start-redis` script | ||
in the scripts directory of the repository can be used to start Redis: | ||
|
||
```bash | ||
$ cd pulse | ||
$ scripts/start-redis | ||
``` | ||
|
||
Then, in two separate terminals, run the following commands: | ||
|
||
```bash | ||
$ source .env | ||
$ go run examples/pool/worker/main.go | ||
``` | ||
|
||
The above start two workers that wait for jobs. Then, in separate terminals, run | ||
the following commands: | ||
|
||
```bash | ||
$ source .env | ||
$ go run examples/pool/producer/main.go | ||
``` | ||
|
||
The above starts and stops two jobs. The first job is handled by the first worker, | ||
and the second job is handled by the second worker. | ||
|
||
Finally in the same terminal used above run the following command: | ||
|
||
```bash | ||
$ go run examples/pool/scheduler/main.go | ||
``` | ||
|
||
The above starts a scheduler that starts and stops jobs alternately. The first job | ||
is handled by the first worker, and the second job is handled by the second worker. | ||
|
||
To stop the worker processes simply press `Ctrl-C` in the terminal where they are | ||
running. Note that it takes a few seconds for the workers to stop (as they wait for | ||
the Redis poll to return). | ||
|
||
## Adding Verbose Logging | ||
|
||
The three processes can be run with verbose logging by passing the `-v` flag: | ||
|
||
```bash | ||
$ go run examples/pool/worker/main.go -v | ||
$ go run examples/pool/producer/main.go -v | ||
$ go run examples/pool/scheduler/main.go -v | ||
``` | ||
|
||
Verbose logging will print logs created by the `pulse` package to the terminal. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"os" | ||
"time" | ||
|
||
redis "github.com/redis/go-redis/v9" | ||
"goa.design/clue/log" | ||
"goa.design/pulse/pool" | ||
"goa.design/pulse/pulse" | ||
) | ||
|
||
func main() { | ||
// Setup Redis connection | ||
rdb := redis.NewClient(&redis.Options{ | ||
Addr: "localhost:6379", | ||
Password: os.Getenv("REDIS_PASSWORD"), | ||
}) | ||
|
||
// Setup clue logger. | ||
ctx := log.Context(context.Background()) | ||
log.FlushAndDisableBuffering(ctx) | ||
|
||
var logger pulse.Logger | ||
if len(os.Args) > 1 && os.Args[1] == "-v" { | ||
logger = pulse.ClueLogger(ctx) | ||
} | ||
|
||
// Create node for pool "example". | ||
node, err := pool.AddNode(ctx, "example", rdb, pool.WithLogger(logger)) | ||
if err != nil { | ||
panic(err) | ||
} | ||
|
||
// Start schedule | ||
log.Infof(ctx, "Starting schedule... CTRL+C to stop.") | ||
done := make(chan struct{}) | ||
if err := node.Schedule(ctx, newProducer(ctx, done), time.Second); err != nil { | ||
panic(err) | ||
} | ||
|
||
// Wait for producer to be done | ||
<-done | ||
|
||
// Cleanup node on exit. | ||
if err := node.Close(ctx); err != nil { | ||
panic(err) | ||
} | ||
} | ||
|
||
// producer is a job producer that alternatively starts and stops a job. | ||
// It closes the done channel when it is done. | ||
type producer struct { | ||
iter int | ||
done chan struct{} | ||
logctx context.Context | ||
} | ||
|
||
func newProducer(ctx context.Context, done chan struct{}) *producer { | ||
return &producer{done: done, logctx: ctx} | ||
} | ||
|
||
func (p *producer) Name() string { | ||
return "example" | ||
} | ||
|
||
// Plan is called by the scheduler to determine the next job to start or stop. | ||
func (p *producer) Plan() (*pool.JobPlan, error) { | ||
p.iter++ | ||
if p.iter > 10 { | ||
log.Infof(p.logctx, "done") | ||
close(p.done) | ||
return nil, pool.ErrScheduleStop | ||
} | ||
if p.iter%2 == 0 { | ||
log.Infof(p.logctx, "stop all") | ||
return &pool.JobPlan{StopAll: true}, nil | ||
} | ||
log.Infof(p.logctx, "start job") | ||
return &pool.JobPlan{Start: []*pool.JobParam{{Key: "job", Payload: []byte("payload")}}}, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.