feat(hatchery/swarm): --add-host options (#2422)

docs/content/hatchery/swarm.md

@@ -30,4 +30,4 @@ This hatchery will now start worker of model 'docker' on you docker installation
 
 ## Setup a worker model
 
-See [Tutorial]({{< relref "workflows/pipelines/requirements/worker-model/docker-simple.md" >}})
+See [Tutorial]({{< relref "workflows/pipelines/requirements/worker-model/docker/_index.md" >}})

docs/content/hosting/ready-to-run/docker-compose.md

@@ -102,7 +102,7 @@ $ docker-compose up cds-hatchery-swarm
 A `swarm hatchery` spawns CDS Workers inside dedicated containers.
 This ensures isolation of the workspaces and resources.
 
-Now, you have to create worker model of type `docker`, please follow [how to create a worker model docker]({{< relref "workflows/pipelines/requirements/worker-model/docker-simple.md" >}}).
+Now, you have to create worker model of type `docker`, please follow [how to create a worker model docker]({{< relref "workflows/pipelines/requirements/worker-model/docker/_index.md" >}}).
 
 ## Next with Actions, Plugins
 

docs/content/workflows/pipelines/requirements/service/example-pg.md

@@ -18,7 +18,7 @@ POSTGRES_USER=myuser
 POSTGRES_PASSWORD=mypassword
 ```
 
-And a requirement model which allow you to execute `apt-get install -y postgresql-client`, see [HowTo]({{< relref "workflows/pipelines/requirements/worker-model/docker-simple.md" >}})
+And a requirement model which allow you to execute `apt-get install -y postgresql-client`, see [HowTo]({{< relref "workflows/pipelines/requirements/worker-model/docker/_index.md" >}})
 
 
 ![Requirement](/images/tutorials_service_link_pg_requirements.png)

docs/content/workflows/pipelines/requirements/worker-model/_index.md

@@ -15,7 +15,7 @@ The goal of a worker model is to describe the capabilities of a given docker/iso
 
 There are 2 types of worker models:
 
- * Docker images, see [how to create a worker model docker]({{< relref "workflows/pipelines/requirements/worker-model/docker-simple.md" >}})
+ * Docker images, see [how to create a worker model docker]({{< relref "workflows/pipelines/requirements/worker-model/docker/_index.md" >}})
  * Openstack images, see [how to create a worker model openstack]({{< relref "workflows/pipelines/requirements/worker-model/openstack.md" >}})
 
 ### Behavior

docs/content/workflows/pipelines/requirements/worker-model/docker/_index.md

@@ -0,0 +1,36 @@
++++
+title = "Docker Worker Model"
+weight = 1
+
++++
+
+A worker model of type `docker` can be spawned by a Hatchery Docker Swarm or a Hatchery Marathon.
+
+## Register a worker Model from an existing Docker Image
+
+Docker Image *golang:1.8.1* have a "curl" in $PATH, so it can be used as it is.
+
+* In the UI, click on the wheel on the hand right top corner and select *workers" (or go the the route *#/worker*)
+* At the bottom of the page, fill the form
+    * Name of your worker *Golang-1.8.1*
+    * type *docker*
+    * image *golang:1.8.1*
+* Click on *Add* button and that's it
+
+![Add worker model](/images/workflows.pipelines.requirements.docker.worker-model.docker.add.png)
+
+## Worker Model Docker on Hatchery Swarm
+
+This hatchery offers some features on job pre-requisites, usable only on user's hatchery (ie. not a shared.infra hatchery).
+
+* [Service Link]({{< relref "workflows/pipelines/requirements/service/_index.md" >}})
+* options on worker model prerequisite
+    * Port mapping: `--port=8080:8081/tcp --port=9080:9081/tcp`
+    * Priviledge flag: `--privileged`
+    * Add host flag: `--add-host=aaa:1.2.3.4 --add-host=bbb:5.6.7.8`
+    * Use all: `--port=8080:8081/tcp --privileged --port=9080:9081/tcp --add-host=aaa:1.2.3.4 --add-host=bbb:5.6.7.8`
+* options on volume prerequisite
+    * Bind: `type=bind,source=/hostDir/sourceDir,destination=/dirInJob,readonly`
+
+![Job Prerequisites](/images/workflows.pipelines.requirements.docker.worker-model.docker.png)
+

engine/hatchery/swarm/swarm.go

@@ -73,13 +73,10 @@ func (h *HatcherySwarm) SpawnWorker(spawnArgs hatchery.SpawnArguments) (string,
 
 	log.Debug("SpawnWorker> Spawning worker %s - %s", name, spawnArgs.LogInfo)
 
-	//Create a network
-	network := name + "-net"
-	h.createNetwork(network)
-
 	//Memory for the worker
 	memory := int64(h.Config.DefaultMemory)
 
+	var network, networkAlias string
 	services := []string{}
 
 	if spawnArgs.JobID > 0 {
@@ -92,6 +89,13 @@ func (h *HatcherySwarm) SpawnWorker(spawnArgs hatchery.SpawnArguments) (string,
 					return "", err
 				}
 			} else if r.Type == sdk.ServiceRequirement {
+				//Create a network if not already created
+				if network == "" {
+					network = name + "-net"
+					networkAlias = "worker"
+					h.createNetwork(network)
+				}
+
 				//name= <alias> => the name of the host put in /etc/hosts of the worker
 				//value= "postgres:latest env_1=blabla env_2=blabla"" => we can add env variables in requirement name
 				tuple := strings.Split(r.Value, " ")
@@ -135,7 +139,7 @@ func (h *HatcherySwarm) SpawnWorker(spawnArgs hatchery.SpawnArguments) (string,
 					entryPoint:   nil,
 				}
 
-				if err := h.createAndStartContainer(args); err != nil {
+				if err := h.createAndStartContainer(args, spawnArgs); err != nil {
 					log.Warning("SpawnWorker>Unable to start required container: %s", err)
 					return "", err
 				}
@@ -207,7 +211,7 @@ func (h *HatcherySwarm) SpawnWorker(spawnArgs hatchery.SpawnArguments) (string,
 		name:         name,
 		image:        spawnArgs.Model.Image,
 		network:      network,
-		networkAlias: "worker",
+		networkAlias: networkAlias,
 		cmd:          cmd,
 		env:          env,
 		labels:       labels,
@@ -217,7 +221,7 @@ func (h *HatcherySwarm) SpawnWorker(spawnArgs hatchery.SpawnArguments) (string,
 	}
 
 	//start the worker
-	if err := h.createAndStartContainer(args); err != nil {
+	if err := h.createAndStartContainer(args, spawnArgs); err != nil {
 		log.Warning("SpawnWorker> Unable to start container named %s with image %s err:%s", name, spawnArgs.Model.Image, err)
 	}
 
@@ -458,6 +462,11 @@ func (h *HatcherySwarm) listAwolWorkers() ([]types.Container, error) {
 	//Checking workers
 	oldContainers := []types.Container{}
 	for _, c := range containers {
+		if time.Now().Add(-1*time.Minute).Unix() < c.Created {
+			log.Debug("listAwolWorkers> container %s is too young", c.Names[0])
+			continue
+		}
+
 		//If there isn't any worker registered on the API. Kill the container
 		if len(apiworkers) == 0 {
 			oldContainers = append(oldContainers, c)
@@ -483,6 +492,7 @@ func (h *HatcherySwarm) listAwolWorkers() ([]types.Container, error) {
 		}
 	}
 
+	log.Debug("listAwolWorkers> oldContainers: %d", len(oldContainers))
 	return oldContainers, nil
 }
 
@@ -514,18 +524,14 @@ func (h *HatcherySwarm) killAwolWorker() error {
 		}
 		//check if the service is linked to a worker which doesn't exist
 		if w, _ := h.getContainer(c.Labels["service_worker"], types.ContainerListOptions{All: true}); w == nil {
-			oldContainers = append(oldContainers, c)
+			log.Debug("killAwolWorker> Delete worker (service) %s", c.Names[0])
+			if err := h.killAndRemove(c.ID); err != nil {
+				log.Error("killAwolWorker> service %v", err)
+			}
 			continue
 		}
 	}
 
-	for _, c := range oldContainers {
-		log.Debug("killAwolWorker> Delete worker %s", c.Names[0])
-		if err := h.killAndRemove(c.ID); err != nil {
-			log.Error("killAwolWorker> %v", err)
-		}
-	}
-
 	return h.killAwolNetworks()
 }
 

engine/hatchery/swarm/swarm_util_create.go

@@ -14,6 +14,7 @@ import (
 	context "golang.org/x/net/context"
 
 	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/hatchery"
 	"github.com/ovh/cds/sdk/log"
 )
 
@@ -45,15 +46,14 @@ type containerArgs struct {
 }
 
 //shortcut to create+start(=run) a container
-func (h *HatcherySwarm) createAndStartContainer(cArgs containerArgs) error {
+func (h *HatcherySwarm) createAndStartContainer(cArgs containerArgs, spawnArgs hatchery.SpawnArguments) error {
 	//Memory is set to 1GB by default
 	if cArgs.memory <= 4 {
 		cArgs.memory = 1024
 	}
 	log.Debug("createAndStartContainer> Create container %s from %s on network %s as %s (memory=%dMB)", cArgs.name, cArgs.image, cArgs.network, cArgs.networkAlias, cArgs.memory)
 
 	var exposedPorts nat.PortSet
-	var mounts []mount.Mount
 
 	name := cArgs.name
 	config := &container.Config{
@@ -71,7 +71,8 @@ func (h *HatcherySwarm) createAndStartContainer(cArgs containerArgs) error {
 	hostConfig := &container.HostConfig{
 		PortBindings: cArgs.dockerOpts.ports,
 		Privileged:   cArgs.dockerOpts.privileged,
-		Mounts:       mounts,
+		Mounts:       cArgs.dockerOpts.mounts,
+		ExtraHosts:   cArgs.dockerOpts.extraHosts,
 	}
 	hostConfig.Resources = container.Resources{
 		Memory:     cArgs.memory * 1024 * 1024, //from MB to B
@@ -88,13 +89,40 @@ func (h *HatcherySwarm) createAndStartContainer(cArgs containerArgs) error {
 		}
 	}
 
+	// ensure that image exists for register
+	if spawnArgs.RegisterOnly {
+		var images []types.ImageSummary
+		var errl error
+		images, errl = h.dockerClient.ImageList(context.Background(), types.ImageListOptions{All: true})
+		if errl != nil {
+			log.Warning("createAndStartContainer> Unable to list images: %s", errl)
+		}
+
+		var imageFound bool
+	checkImage:
+		for _, img := range images {
+			for _, t := range img.RepoTags {
+				if cArgs.image == t {
+					imageFound = true
+					break checkImage
+				}
+			}
+		}
+
+		if !imageFound {
+			if err := h.pullImage(cArgs.image, timeoutPullImage); err != nil {
+				return sdk.WrapError(err, "createAndStartContainer> Unable to pull image %s", cArgs.image)
+			}
+		}
+	}
+
 	c, err := h.dockerClient.ContainerCreate(context.Background(), config, hostConfig, networkingConfig, name)
 	if err != nil {
-		return sdk.WrapError(err, "startAndCreateContainer> Unable to create container %s", name)
+		return sdk.WrapError(err, "createAndStartContainer> Unable to create container %s", name)
 	}
 
 	if err := h.dockerClient.ContainerStart(context.Background(), c.ID, types.ContainerStartOptions{}); err != nil {
-		return sdk.WrapError(err, "startAndCreateContainer> Unable to start container %v %s", c.ID[:12])
+		return sdk.WrapError(err, "createAndStartContainer> Unable to start container %v %s", c.ID[:12])
 	}
 	return nil
 }
@@ -105,6 +133,7 @@ type dockerOpts struct {
 	ports      nat.PortMap
 	privileged bool
 	mounts     []mount.Mount
+	extraHosts []string
 }
 
 func computeDockerOpts(isSharedInfra bool, requirements []sdk.Requirement) (*dockerOpts, error) {
@@ -140,6 +169,10 @@ func (d *dockerOpts) computeDockerOptsOnModelRequirement(isSharedInfra bool, req
 			if err := d.computeDockerOptsPorts(opt); err != nil {
 				return err
 			}
+		} else if strings.HasPrefix(opt, "--add-host=") {
+			if err := d.computeDockerOptsExtraHosts(opt); err != nil {
+				return err
+			}
 		} else if opt == "--privileged" {
 			d.privileged = true
 		} else {
@@ -215,6 +248,12 @@ func (d *dockerOpts) computeDockerOptsOnVolumeMountRequirement(opt string) error
 	return nil
 }
 
+func (d *dockerOpts) computeDockerOptsExtraHosts(arg string) error {
+	value := strings.TrimPrefix(strings.TrimSpace(arg), "--add-host=")
+	d.extraHosts = append(d.extraHosts, value)
+	return nil
+}
+
 func (d *dockerOpts) computeDockerOptsPorts(arg string) error {
 	if regexPort.MatchString(arg) {
 		s := regexPort.FindStringSubmatch(arg)

engine/hatchery/swarm/swarm_util_create_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/ovh/cds/engine/api/test"
 	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/hatchery"
 )
 
 func Test_computeDockerOpts(t *testing.T) {
@@ -103,6 +104,19 @@ func Test_computeDockerOpts(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "Extra hosts",
+			args: args{requirements: []sdk.Requirement{{Name: "go-official-1.9.1", Type: sdk.ModelRequirement, Value: "golang:1.9.1 --port=8080:8081/tcp --privileged --port=9080:9081/tcp --add-host=aaa:1.2.3.4 --add-host=bbb:5.6.7.8"}}},
+			want: &dockerOpts{
+				privileged: true,
+				ports: nat.PortMap{
+					nat.Port("8081/tcp"): []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: "8080"}},
+					nat.Port("9081/tcp"): []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: "9080"}},
+				},
+				extraHosts: []string{"aaa:1.2.3.4", "bbb:5.6.7.8"},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -128,10 +142,9 @@ func TestHatcherySwarm_createAndStartContainer(t *testing.T) {
 		memory: 256,
 	}
 
-	err := h.pullImage(args.image, timeoutPullImage)
-	test.NoError(t, err)
-
-	err = h.createAndStartContainer(args)
+	// RegisterOnly = true, this will pull image if image is not found
+	spawnArgs := hatchery.SpawnArguments{RegisterOnly: true}
+	err := h.createAndStartContainer(args, spawnArgs)
 	test.NoError(t, err)
 
 	cntr, err := h.getContainer(args.name, types.ContainerListOptions{})
@@ -168,7 +181,8 @@ func TestHatcherySwarm_createAndStartContainerWithMount(t *testing.T) {
 	err := h.pullImage(args.image, timeoutPullImage)
 	test.NoError(t, err)
 
-	err = h.createAndStartContainer(args)
+	spawnArgs := hatchery.SpawnArguments{RegisterOnly: false}
+	err = h.createAndStartContainer(args, spawnArgs)
 	test.NoError(t, err)
 
 	cntr, err := h.getContainer(args.name, types.ContainerListOptions{})
@@ -194,7 +208,8 @@ func TestHatcherySwarm_createAndStartContainerWithNetwork(t *testing.T) {
 	err := h.createNetwork(args.network)
 	test.NoError(t, err)
 
-	err = h.createAndStartContainer(args)
+	spawnArgs := hatchery.SpawnArguments{RegisterOnly: false}
+	err = h.createAndStartContainer(args, spawnArgs)
 	test.NoError(t, err)
 
 	cntr, err := h.getContainer(args.name, types.ContainerListOptions{})

engine/hatchery/swarm/swarm_util_kill.go

@@ -79,14 +79,14 @@ func (h *HatcherySwarm) killAwolNetworks() error {
 	//Checking networks
 	nets, errLN := h.dockerClient.NetworkList(context.Background(), types.NetworkListOptions{})
 	if errLN != nil {
-		log.Warning("killAwolWorker> Cannot get networks: %s", errLN)
+		log.Warning("killAwolNetworks> Cannot get networks: %s", errLN)
 		return errLN
 	}
 
 	for i := range nets {
 		n, err := h.dockerClient.NetworkInspect(context.Background(), nets[i].ID)
 		if err != nil {
-			log.Warning("killAwolWorker> Unable to get network info: %v", err)
+			log.Warning("killAwolNetworks> Unable to get network info: %v", err)
 			continue
 		}
 
@@ -102,9 +102,9 @@ func (h *HatcherySwarm) killAwolNetworks() error {
 			continue
 		}
 
-		log.Debug("killAwolWorker> Delete network %s", n.Name)
+		log.Debug("killAwolNetworks> Delete network %s", n.Name)
 		if err := h.dockerClient.NetworkRemove(context.Background(), n.ID); err != nil {
-			log.Warning("killAwolWorker> Unable to delete network %s err:%s", n.Name, err)
+			log.Warning("killAwolNetworks> Unable to delete network %s err:%s", n.Name, err)
 		}
 	}
 	return nil

engine/hatchery/swarm/swarm_util_pull.go

@@ -25,8 +25,5 @@ func (h *HatcherySwarm) pullImage(img string, timeout time.Duration) error {
 
 	btes, _ := ioutil.ReadAll(res)
 	log.Debug("pullImage> %s", string(btes))
-	if err := res.Close(); err != nil {
-		return err
-	}
-	return nil
+	return res.Close()
 }