docker.go

// Bivac v2.0.0 (https://camptocamp.github.io/bivac)
// Copyright (c) 2019 Camptocamp
// Licensed under Apache-2.0 (https://raw.githubusercontent.com/camptocamp/bivac/master/LICENSE)
// Modifications copyright (c) 2019 Jam Risser <jam@codejam.ninja>

package orchestrators

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"unicode/utf8"

	"github.com/docker/docker/api/types"
	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/mount"
	docker "github.com/docker/docker/client"
	"github.com/docker/docker/pkg/stdcopy"
	"golang.org/x/net/context"

	"github.com/codejamninja/volback/pkg/volume"
)

// DockerConfig stores Docker configuration
type DockerConfig struct {
	Endpoint string
}

// DockerOrchestrator implements a container orchestrator for Docker
type DockerOrchestrator struct {
	client docker.CommonAPIClient
}

// NewDockerOrchestrator creates a Docker client
func NewDockerOrchestrator(config *DockerConfig) (o *DockerOrchestrator, err error) {
	o = &DockerOrchestrator{}
	o.client, err = docker.NewClient(config.Endpoint, "", nil, nil)
	if err != nil {
		err = fmt.Errorf("failed to create a Docker client: %s", err)
	}
	return
}

// GetName returns the orchestrator's name
func (o *DockerOrchestrator) GetName() string {
	return "docker"
}

// GetPath returns the backup path
func (*DockerOrchestrator) GetPath(v *volume.Volume) string {
	return v.Hostname
}

// GetVolumes returns the Docker volumes, inspected and filtered
func (o *DockerOrchestrator) GetVolumes(volumeFilters volume.Filters) (volumes []*volume.Volume, err error) {

	info, err := o.client.Info(context.Background())
	if err != nil {
		err = fmt.Errorf("failed to retrieve Docker engine info: %s", err)
		return
	}

	vols, err := o.client.VolumeList(context.Background(), filters.NewArgs())
	if err != nil {
		err = fmt.Errorf("failed to list Docker volumes: %v", err)
		return
	}

	var voll types.Volume
	for _, vol := range vols.Volumes {
		voll, err = o.client.VolumeInspect(context.Background(), vol.Name)
		if err != nil {
			err = fmt.Errorf("failed to inspect volume `%s': %v", vol.Name, err)
			return
		}

		v := &volume.Volume{
			ID:         voll.Name,
			Name:       voll.Name,
			Mountpoint: voll.Mountpoint,
			HostBind:   info.Name,
			Hostname:   info.Name,
			Labels:     voll.Labels,
			Logs:       make(map[string]string),
			RepoName:   voll.Name,
			SubPath:    "",
		}

		if b, _, _ := o.blacklistedVolume(v, volumeFilters); b {
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

// DeployAgent creates a `volback agent` container
func (o *DockerOrchestrator) DeployAgent(image string, cmd []string, envs []string, v *volume.Volume) (success bool, output string, err error) {
	success = false
	err = o.PullImage(image)
	if err != nil {
		err = fmt.Errorf("failed to pull image: %s", err)
		return
	}

	additionalVolumes, err := o.getAdditionalVolumes()
	if err != nil {
		err = fmt.Errorf("failed to get additional volumes: %s", err)
		return
	}

	mounts := []mount.Mount{
		mount.Mount{
			Type:     "volume",
			Target:   v.Mountpoint,
			Source:   v.Name,
			ReadOnly: v.ReadOnly,
		},
	}

	mounts = append(mounts, additionalVolumes...)

	container, err := o.client.ContainerCreate(
		context.Background(),
		&containertypes.Config{
			Hostname:     createAgentName(),
			Cmd:          cmd,
			Env:          envs,
			Image:        image,
			OpenStdin:    true,
			StdinOnce:    true,
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			Tty:          false,
		},
		&containertypes.HostConfig{
			Mounts: mounts,
		}, nil, "")

	if err != nil {
		err = fmt.Errorf("failed to create container: %s", err)
		return
	}
	defer o.RemoveContainer(container.ID)

	err = o.client.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
	if err != nil {
		err = fmt.Errorf("failed to start container: %s", err)
		return
	}

	var exited bool

	for !exited {
		var cont types.ContainerJSON
		cont, err = o.client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			err = fmt.Errorf("failed to inspect container: %s", err)
			return
		}

		if cont.State.Status == "exited" {
			exited = true
		}
	}

	body, err := o.client.ContainerLogs(context.Background(), container.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Details:    true,
		Follow:     true,
	})
	if err != nil {
		err = fmt.Errorf("failed to retrieve logs: %s", err)
		return
	}

	defer body.Close()
	stdout := new(bytes.Buffer)
	_, err = stdcopy.StdCopy(stdout, ioutil.Discard, body)
	if err != nil {
		err = fmt.Errorf("failed to read logs from response: %s", err)
		return
	}
	logs := strings.Split(stdout.String(), "\n")
	if len(logs) > 1 {
		output = logs[len(logs)-2]
	}
	success = true
	return
}

// RemoveContainer removes a container based on its ID
func (o *DockerOrchestrator) RemoveContainer(containerID string) (err error) {
	err = o.client.ContainerRemove(context.Background(), containerID, types.ContainerRemoveOptions{
		Force:         true,
		RemoveVolumes: true,
	})
	return
}

// DetectDocker tries to detect a Docker orchestrator by connecting to the endpoint
func DetectDocker(config *DockerConfig) bool {
	client, err := docker.NewClient(config.Endpoint, "", nil, nil)
	if err != nil {
		return false
	}
	_, err = client.Ping(context.Background())
	if err != nil {
		return false
	}
	return true
}

// PullImage pulls a Docker image
func (o *DockerOrchestrator) PullImage(image string) (err error) {
	if _, _, err = o.client.ImageInspectWithRaw(context.Background(), image); err != nil {
		resp, err := o.client.ImagePull(context.Background(), image, types.ImagePullOptions{})
		if err != nil {
			return err
		}
		defer resp.Close()

		_, err = ioutil.ReadAll(resp)
		if err != nil {
			err = fmt.Errorf("failed to read ImagePull response: %s", err)
			return err
		}
	}
	return nil
}

// GetContainersMountingVolume returns mounted volumes
func (o *DockerOrchestrator) GetContainersMountingVolume(v *volume.Volume) (mountedVolumes []*volume.MountedVolume, err error) {
	c, err := o.client.ContainerList(context.Background(), types.ContainerListOptions{})
	if err != nil {
		err = fmt.Errorf("failed to list containers: %s", err)
		return
	}

	for _, container := range c {
		for _, mount := range container.Mounts {
			if mount.Name == v.Name && mount.Type == "volume" {
				mv := &volume.MountedVolume{
					ContainerID: container.ID,
					Volume:      v,
					Path:        mount.Destination,
				}
				mountedVolumes = append(mountedVolumes, mv)
			}
		}
	}
	return
}

// ContainerExec executes a command in a container
func (o *DockerOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolume, command []string) (stdout string, err error) {
	exec, err := o.client.ContainerExecCreate(context.Background(), mountedVolumes.ContainerID, types.ExecConfig{
		AttachStdout: true,
		AttachStderr: true,
		Cmd:          command,
	})
	if err != nil {
		err = fmt.Errorf("failed to create exec: %s", err)
		return
	}

	conn, err := o.client.ContainerExecAttach(context.Background(), exec.ID, types.ExecStartCheck{})
	if err != nil {
		err = fmt.Errorf("failed to attach: %s", err)
		return
	}
	defer conn.Close()

	err = o.client.ContainerExecStart(context.Background(), exec.ID, types.ExecStartCheck{})
	if err != nil {
		err = fmt.Errorf("failed to start exec: %s", err)
		return
	}

	stdoutput := new(bytes.Buffer)
	stdcopy.StdCopy(stdoutput, ioutil.Discard, conn.Reader)

	stdout = stdoutput.String()
	return
}

// IsNodeAvailable checks if the node is available to run backups on it
func (o *DockerOrchestrator) IsNodeAvailable(hostID string) (ok bool, err error) {
	// We can assume that, if Volback is running then, the Docker daemon is available
	ok = true
	return
}

// RetrieveOrphanAgents returns the list of orphan Volback agents
func (o *DockerOrchestrator) RetrieveOrphanAgents() (containers map[string]string, err error) {
	c, err := o.client.ContainerList(context.Background(), types.ContainerListOptions{})
	if err != nil {
		err = fmt.Errorf("failed to list containers: %s", err)
		return
	}

	for _, container := range c {
		if !strings.HasPrefix(container.ID, "volback-agent-") {
			continue
		}

		for _, mount := range container.Mounts {
			if mount.Type == "volume" {
				containers[mount.Name] = container.ID
			}
		}
	}
	return
}

// AttachOrphanAgent connects to a running agent and wait for the end of the backup proccess
func (o *DockerOrchestrator) AttachOrphanAgent(containerID, namespace string) (success bool, output string, err error) {
	container, err := o.client.ContainerInspect(context.Background(), containerID)
	if err != nil {
		err = fmt.Errorf("failed to inspect container: %s", err)
		return
	}

	defer o.RemoveContainer(container.ID)

	err = o.client.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
	if err != nil {
		err = fmt.Errorf("failed to start container: %s", err)
		return
	}

	var exited bool

	for !exited {
		var cont types.ContainerJSON
		cont, err = o.client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			err = fmt.Errorf("failed to inspect container: %s", err)
			return
		}

		if cont.State.Status == "exited" {
			exited = true
		}
	}

	body, err := o.client.ContainerLogs(context.Background(), container.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Details:    true,
		Follow:     true,
	})
	if err != nil {
		err = fmt.Errorf("failed to retrieve logs: %s", err)
		return
	}

	defer body.Close()
	stdout := new(bytes.Buffer)
	_, err = stdcopy.StdCopy(stdout, ioutil.Discard, body)
	if err != nil {
		err = fmt.Errorf("failed to read logs from response: %s", err)
		return
	}
	logs := strings.Split(stdout.String(), "\n")
	if len(logs) > 1 {
		output = logs[len(logs)-2]
	}
	success = true
	return
}

func (o *DockerOrchestrator) blacklistedVolume(vol *volume.Volume, volumeFilters volume.Filters) (bool, string, string) {
	if utf8.RuneCountInString(vol.Name) == 64 || vol.Name == "lost+found" {
		return true, "unnamed", ""
	}

	// Check labels
	if ignored, ok := vol.Labels["volback.ignore"]; ok && ignored == "true" {
		return true, "ignored", "volume config"
	}

	// Use whitelist if defined
	if l := volumeFilters.Whitelist; len(l) > 0 && l[0] != "" {
		sort.Strings(l)
		i := sort.SearchStrings(l, vol.Name)
		if i < len(l) && l[i] == vol.Name {
			return false, "", ""
		}
		return true, "blacklisted", "whitelist config"
	}

	i := sort.SearchStrings(volumeFilters.Blacklist, vol.Name)
	if i < len(volumeFilters.Blacklist) && volumeFilters.Blacklist[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}
	return false, "", ""
}

func (o *DockerOrchestrator) getAdditionalVolumes() (mounts []mount.Mount, err error) {
	mounts = []mount.Mount{}

	managerHostname, err := os.Hostname()
	if err != nil {
		err = fmt.Errorf("failed to retrieve hostname: %s", err)
		return
	}

	managerContainer, err := o.client.ContainerInspect(context.Background(), managerHostname)
	if err != nil {
		if strings.Contains(err.Error(), "No such container") {
			// We assume Volback is running outside the orchestrator
			err = nil
		} else {
			err = fmt.Errorf("failed to inspect container: %s", err)
		}
		return
	}

	for _, v := range managerContainer.Mounts {
		mounts = append(mounts, mount.Mount{
			Type:     v.Type,
			Source:   v.Source,
			Target:   v.Destination,
			ReadOnly: !v.RW,
		})
	}

	return
}