-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fixed size worker pool to serve new job requests (#72)
* X amount of goroutines listen to a queue for new job requests * HTTP new job requests place the work into the queue * the result is communicated back to the HTTP request goroutine through a per-request unique result channel
- Loading branch information
Showing
11 changed files
with
263 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,5 +3,7 @@ | |
"build_path": "/tmp", | ||
"mounts": { | ||
"/tmp": "/tmp" | ||
} | ||
}, | ||
"job_concurrency": 5, | ||
"job_backlog": 100 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
FROM debian:stretch | ||
|
||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh | ||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh | ||
|
||
WORKDIR /data | ||
|
||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
sleep 10 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
"log" | ||
"sync" | ||
|
||
"github.com/skroutz/mistry/pkg/types" | ||
) | ||
|
||
// WorkResult contains the result of a job, either a buildinfo or an error | ||
type WorkResult struct { | ||
BuildInfo *types.BuildInfo | ||
Err error | ||
} | ||
|
||
// FutureWorkResult is a WorkResult that may not yet have become available and | ||
// can be Wait()'ed on | ||
type FutureWorkResult struct { | ||
result <-chan WorkResult | ||
} | ||
|
||
// Wait waits for the result to become available and returns it | ||
func (f FutureWorkResult) Wait() WorkResult { | ||
r, ok := <-f.result | ||
if !ok { | ||
// this should never happen, reading from the result channel is exclusive to | ||
// this future | ||
panic("Failed to read from result channel") | ||
} | ||
return r | ||
} | ||
|
||
// workItem contains a job and a channel to place the job result. struct | ||
// used in the internal work queue | ||
type workItem struct { | ||
job *Job | ||
result chan<- WorkResult | ||
} | ||
|
||
// WorkerPool implements a fixed-size pool of worker goroutines that can be sent | ||
// build jobs and communicate their result | ||
type WorkerPool struct { | ||
// the fixed amount of goroutines that will be handling running jobs | ||
concurrency int | ||
|
||
// the maximum backlog of pending requests. if exceeded, sending new work | ||
// to the pool will return an error | ||
backlogSize int | ||
|
||
queue chan workItem | ||
wg sync.WaitGroup | ||
} | ||
|
||
// NewWorkerPool creates a new worker pool | ||
func NewWorkerPool(s *Server, concurrency, backlog int, logger *log.Logger) *WorkerPool { | ||
p := new(WorkerPool) | ||
p.concurrency = concurrency | ||
p.backlogSize = backlog | ||
p.queue = make(chan workItem, backlog) | ||
|
||
for i := 0; i < concurrency; i++ { | ||
go work(s, i, p.queue, &p.wg) | ||
p.wg.Add(1) | ||
} | ||
logger.Printf("Set up %d workers", concurrency) | ||
return p | ||
} | ||
|
||
// Stop signals the workers to close and blocks until they are closed. | ||
func (p *WorkerPool) Stop() { | ||
close(p.queue) | ||
p.wg.Wait() | ||
} | ||
|
||
// SendWork schedules work on p and returns a FutureWorkResult. The actual result can be | ||
// obtained by FutureWorkResult.Wait(). An error is returned if the backlog is full and | ||
// cannot accept any new work items | ||
func (p *WorkerPool) SendWork(j *Job) (FutureWorkResult, error) { | ||
resultQueue := make(chan WorkResult, 1) | ||
wi := workItem{j, resultQueue} | ||
result := FutureWorkResult{resultQueue} | ||
|
||
select { | ||
case p.queue <- wi: | ||
return result, nil | ||
default: | ||
return result, errors.New("queue is full") | ||
} | ||
} | ||
|
||
// work listens to the workQueue, runs Work() on any incoming work items, and | ||
// sends the result through the result queue | ||
func work(s *Server, id int, queue <-chan workItem, wg *sync.WaitGroup) { | ||
defer wg.Done() | ||
logPrefix := fmt.Sprintf("[worker %d]", id) | ||
for item := range queue { | ||
s.Log.Printf("%s received work item %#v", logPrefix, item) | ||
buildInfo, err := s.Work(context.Background(), item.job) | ||
|
||
select { | ||
case item.result <- WorkResult{buildInfo, err}: | ||
s.Log.Printf("%s wrote result to the result channel", logPrefix) | ||
default: | ||
// this should never happen, the result chan should be unique for this worker | ||
s.Log.Panicf("%s failed to write result to the result channel", logPrefix) | ||
} | ||
close(item.result) | ||
} | ||
s.Log.Printf("%s exiting...", logPrefix) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
package main | ||
|
||
import ( | ||
"testing" | ||
"time" | ||
|
||
"github.com/skroutz/mistry/pkg/types" | ||
) | ||
|
||
func TestBacklogLimit(t *testing.T) { | ||
wp, cfg := setupQueue(t, 1, 0) | ||
defer wp.Stop() | ||
|
||
params := types.Params{"test": "pool-backlog-limit"} | ||
params2 := types.Params{"test": "pool-backlog-limit2"} | ||
project := "simple" | ||
|
||
sendWorkNoErr(wp, project, params, cfg, t) | ||
_, _, err := sendWork(wp, project, params2, cfg, t) | ||
|
||
if err == nil { | ||
t.Fatal("Expected error") | ||
} | ||
} | ||
|
||
func TestConcurrency(t *testing.T) { | ||
// instatiate server with 1 worker | ||
wp, cfg := setupQueue(t, 1, 100) | ||
defer wp.Stop() | ||
|
||
project := "sleep" | ||
params := types.Params{"test": "pool-concurrency"} | ||
params2 := types.Params{"test": "pool-concurrency2"} | ||
|
||
sendWorkNoErr(wp, project, params, cfg, t) | ||
// give the chance for the worker to start work | ||
time.Sleep(1 * time.Second) | ||
|
||
j, _ := sendWorkNoErr(wp, project, params2, cfg, t) | ||
|
||
// the queue should contain only 1 item, the work item for the 2nd job | ||
assertEq(len(wp.queue), 1, t) | ||
select { | ||
case i, ok := <-wp.queue: | ||
if !ok { | ||
t.Fatalf("Unexpectedly closed worker pool queue") | ||
} | ||
assertEq(i.job, j, t) | ||
default: | ||
t.Fatalf("Expected to find a work item in the queue") | ||
} | ||
} | ||
|
||
func setupQueue(t *testing.T, workers, backlog int) (*WorkerPool, *Config) { | ||
cfg := testcfg | ||
cfg.Concurrency = workers | ||
cfg.Backlog = backlog | ||
|
||
s, err := NewServer(cfg, nil) | ||
failIfError(err, t) | ||
return s.workerPool, cfg | ||
} | ||
|
||
func sendWork(wp *WorkerPool, project string, params types.Params, cfg *Config, t *testing.T) (*Job, FutureWorkResult, error) { | ||
j, err := NewJob(project, params, "", cfg) | ||
failIfError(err, t) | ||
|
||
r, err := wp.SendWork(j) | ||
return j, r, err | ||
} | ||
|
||
func sendWorkNoErr(wp *WorkerPool, project string, params types.Params, cfg *Config, t *testing.T) (*Job, FutureWorkResult) { | ||
j, r, err := sendWork(wp, project, params, cfg, t) | ||
failIfError(err, t) | ||
return j, r | ||
} |