This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

minion: Build custom Docker images in parallel

Previously, custom Docker images would be built sequentially by the
master, potentially not making full use of the machine. This change
allows up to 8 images to be built simultaneously.
  • Loading branch information...
aegamesi authored and ejj committed Nov 11, 2017
1 parent 1b31dbe commit b1d8cac3e6c217ee8361f24c9537eabb09b99da5
@@ -3,15 +3,23 @@ const infrastructure = require('../../config/infrastructure.js');
const infra = infrastructure.createTestInfrastructure();
for (let workerIndex = 0; workerIndex < infrastructure.nWorker; workerIndex += 1) {
const image = new kelda.Image(`test-custom-image${workerIndex}`,
const images = [];
for (let imageIndex = 0; imageIndex < 8; imageIndex += 1) {
const image = new kelda.Image(`test-custom-image${imageIndex}`,
'FROM alpine\n' +
`RUN echo ${workerIndex} > /dockerfile-id\n` +
`RUN echo ${imageIndex} > /dockerfile-id\n` +
'RUN echo $(cat /dev/urandom | tr -dc \'a-zA-Z0-9\' | ' +
'fold -w 32 | head -n 1) > /image-id');
for (let containerIndex = 0; containerIndex < 2; containerIndex += 1) {
'fold -w 32 | head -n 1) > /image-id\n' +
'RUN sleep 15');
images.push(image);
}
for (let workerIndex = 0; workerIndex < infrastructure.nWorker; workerIndex += 1) {
for (let containerIndex = 0; containerIndex < images.length; containerIndex += 1) {
const container = new kelda.Container(
'bar', image, { command: ['tail', '-f', '/dev/null'] });
'bar', images[containerIndex], { command: ['tail', '-f', '/dev/null'] });
container.deploy(infra);
}
}
@@ -9,6 +9,7 @@ import (
"github.com/kelda/kelda/db"
"github.com/kelda/kelda/integration-tester/util"
"time"
)
func TestCustomImages(t *testing.T) {
@@ -62,6 +63,11 @@ func TestCustomImages(t *testing.T) {
if countErr != nil {
t.Error(countErr)
}
parallelErr := checkBuildParallelized(machines, containers)
if parallelErr != nil {
t.Error(parallelErr)
}
}
func checkReuseImage(dockerfileToImages map[string][]string) error {
@@ -84,10 +90,36 @@ func checkImageCounts(machines []db.Machine, dockerfileCounts map[string]int) er
}
}
for i := 0; i < nWorker; i++ {
if actual := dockerfileCounts[strconv.Itoa(i)]; actual != 2 {
for i := 0; i < len(dockerfileCounts); i++ {
if actual := dockerfileCounts[strconv.Itoa(i)]; actual != nWorker {
return fmt.Errorf("DockerfileID %d had %d containers, "+
"expected %d", i, actual, 2)
"expected %d", i, actual, nWorker)
}
}
return nil
}
func checkBuildParallelized(machines []db.Machine, containers []db.Container) error {
firstContainer := make(map[string]time.Time)
lastContainer := make(map[string]time.Time)
for _, c := range containers {
if t, ok := firstContainer[c.Minion]; !ok || c.Created.Before(t) {
firstContainer[c.Minion] = c.Created
}
if t, ok := lastContainer[c.Minion]; !ok || c.Created.After(t) {
lastContainer[c.Minion] = c.Created
}
}
for _, m := range machines {
first, last := firstContainer[m.PrivateIP], lastContainer[m.PrivateIP]
duration := last.Sub(first)
maxDuration := 15 * time.Second
if duration > maxDuration {
return fmt.Errorf("machine %s has containers that started %d ms"+
" apart, expected <%d", m.CloudID, duration, maxDuration)
}
}
@@ -9,6 +9,7 @@ import (
"github.com/kelda/kelda/db"
"github.com/kelda/kelda/minion/docker"
"sync"
)
/*
@@ -51,24 +52,41 @@ func syncImages(conn db.Conn, dk docker.Client) {
return nil
})
for _, img := range toBuild {
var wg sync.WaitGroup
wg.Add(len(toBuild))
sema := make(chan struct{}, 8)
builder := func(img db.Image) {
sema <- struct{}{}
defer func() {
writeImage(conn, img)
<-sema
wg.Done()
}()
img.Status = db.Building
writeImage(conn, img)
log.WithField("image", img.Name).Info("Building image...")
id, err := updateRegistry(dk, img)
if err != nil {
img.Status = "" // Unset the building status.
writeImage(conn, img)
log.WithError(err).WithField("image", img.Name).
Error("Failed to update registry")
continue
return
}
img.DockerID = id
img.Status = db.Built
writeImage(conn, img)
log.WithField("image", img.Name).Info("Built image.")
}
for _, img := range toBuild {
go builder(img)
}
wg.Wait()
}
func updateRegistry(dk docker.Client, img db.Image) (string, error) {

0 comments on commit b1d8cac

Please sign in to comment.