v0.14.0

.gitignore

@@ -0,0 +1,37 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+__pycache__
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+.idea/

Dockerfile

@@ -1,9 +1,28 @@
-FROM ubuntu:trusty
+FROM tutum/curl:trusty
 MAINTAINER Feng Honglin <hfeng@tutum.co>
 
-ENV CLEAN_PERIOD **None**
-ENV DELAY_TIME **None**
+ADD . /gopath/src/github.com/tutumcloud/image-cleanup
+
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends -y -q git && \
+    mkdir /goroot && \
+    curl -s https://storage.googleapis.com/golang/go1.3.linux-amd64.tar.gz | tar xzf - -C /goroot --strip-components=1 && \ 
+    export GOROOT=/goroot && \
+    export GOPATH=/gopath && \
+    export PATH=$PATH:/goroot/bin && \
+    go get github.com/tutumcloud/image-cleanup && \
+    cp /gopath/bin/* / && \
+    rm -fr /goroot /gopath /var/lib/apt/lists && \
+    apt-get autoremove -y git && \
+    apt-get clean
+
+ENV DOCKER_ROOT_DIR "/var/lib/docker/"
+ENV IMAGE_CLEAN_INTERVAL 1
+ENV IMAGE_CLEAN_DELAYED 1800
+ENV VOLUME_CLEAN_INTERVAL 1800
+ENV IMAGE_LOCKED **None**
 
 ADD run.sh /run.sh
 RUN chmod +x /run.sh
+
 CMD ["/run.sh"]

README.md

@@ -1,20 +1,21 @@
 tutum/image-cleanup
 =========================
 
-```
     docker run -d \
+      --privileged \
       -v /var/run:/var/run:rw \
-      -v /usr/lib/tutum/docker:/usr/bin/docker:r \
-      -e CLEAN_PERIOD=1800 \
-      -e DELAY_TIME=1800 \
-      -e KEEP_IMAGES="ubuntu:trusty, ubuntu:latest" \
+      -v /var/lib/docker:/var/lib/docker:rw \
+      -e DOCKER_ROOT_DIR="/var/lib/docker/" \
+      -e IMAGE_CLEAN_INTERVAL=1 \
+      -e IMAGE_CLEAN_DELAYED=1800 \
+      -e VOLUME_CLEAN_INTERVAL=1800 \
+      -e IMAGE_LOCKED="ubuntu:trusty, tutum/curl:trusty" \
       tutum/image-cleanup
-```
 
 **Arguments**
 
-```
-    CLEAN_PERIOD    how many seconds to run the clean script, 1800 by default.
-    DELAY_TIME      how many seconds delay to remove docker images, 1800 by default.
-    KEEP_IMAGES     A list of Images that will not be cleaned by this container, separated by ","
-```
+    DOCKER_ROOT_DIR         the root path of docker lib, "/var/docker/lib/" by default
+    IMAGE_CLEAN_INTERVAL    how many seconds to clean the images, 1 by default.
+    IMAGE_CLEAN_DELAYED     how many seconds delay to clean docker images, 1800 by default.
+    VOLUME_CLEAN_INTERVAL   how many seconde to clean docker volumes, 1800 by default.
+    IMAGE_LOCKED            A list of Images that will not be cleaned by this container, separated by ","

cleanup.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
+)
+
+var (
+	pDockerHost          = flag.String("dockerHost", "unix:///var/run/docker.sock", "docker host")
+	pImageCleanInterval  = flag.Int("imageCleanInterval", 1, "interval to run image cleanup")
+	pImageCleanDelayed   = flag.Int("imageCleanDelayed", 1800, "delayed time to clean the images")
+	pVolumeCleanInterval = flag.Int("volumeCleanInterval", 1800, "interval to run volume cleanup")
+	pImageLocked         = flag.String("imageLocked", "", "images to avoid being cleaned")
+	pDockerRootDir       = flag.String("dockerRootDir", "/var/lib/docker", "root path of docker lib")
+)
+
+func init() {
+	runtime.GOMAXPROCS(4)
+}
+
+func main() {
+	flag.Parse()
+
+	client, err := getDockerClient(*pDockerHost)
+	if err != nil {
+		log.Fatalf("Docker %s:%s", err, *pDockerHost)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go cleanImages(client)
+	go cleanVolumes(client)
+	wg.Wait()
+}
+
+func getDockerClient(host string) (*docker.Client, error) {
+	client, err := docker.NewClient(host)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func cleanImages(client *docker.Client) {
+	log.Printf("Img Cleanup: the following images will be locked: %s", *pImageLocked)
+	log.Println("Img Cleanup: starting image cleanup ...")
+	for {
+		// imageIdMap[imageID] = isRemovable
+		imageIdMap := make(map[string]bool)
+
+		// Get the image ID list before the cleanup
+		images, err := client.ListImages(docker.ListImagesOptions{All: false})
+		if err != nil {
+			log.Println("Img Cleanup: cannot get images list", err)
+			time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
+			continue
+		}
+
+		for _, image := range images {
+			imageIdMap[image.ID] = true
+		}
+
+		// Get the image IDs used by all the containers
+		containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
+		if err != nil {
+			log.Println("Img Cleanup: cannot get container list", err)
+			time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
+			continue
+		} else {
+			inspect_error := false
+			for _, container := range containers {
+				containerInspect, err := client.InspectContainer(container.ID)
+				if err != nil {
+					inspect_error = true
+					log.Println("Img Cleanup: cannot get container inspect", err)
+					break
+				}
+				imageIdMap[containerInspect.Image] = false
+			}
+			if inspect_error {
+				time.Sleep(time.Duration(*pImageCleanInterval+*pImageCleanDelayed) * time.Second)
+				continue
+			}
+		}
+
+		// Get all the locked image ID
+		if *pImageLocked != "" {
+			lockedImages := strings.Split(*pImageLocked, ",")
+			for _, lockedImage := range lockedImages {
+				imageInspect, err := client.InspectImage(strings.Trim(lockedImage, " "))
+				if err == nil {
+					imageIdMap[imageInspect.ID] = false
+				}
+
+			}
+		}
+
+		// Sleep for the delay time
+		log.Printf("Img Cleanup: wait %d seconds for the cleaning", *pImageCleanDelayed)
+		time.Sleep(time.Duration(*pImageCleanDelayed) * time.Second)
+
+		// Get the image IDs used by all the containers again after the delay time
+		containersDelayed, err := client.ListContainers(docker.ListContainersOptions{All: true})
+		if err != nil {
+			log.Println("Img Cleanup: cannot get container list", err)
+			time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
+			continue
+		} else {
+			inspect_error := false
+			for _, container := range containersDelayed {
+				containerInspect, err := client.InspectContainer(container.ID)
+				if err != nil {
+					inspect_error = true
+					log.Println("Img Cleanup: cannot get container inspect", err)
+					break
+				}
+				imageIdMap[containerInspect.Image] = false
+			}
+			if inspect_error {
+				time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
+				continue
+			}
+		}
+
+		// Remove the unused images
+		counter := 0
+		for id, removable := range imageIdMap {
+			if removable {
+				log.Printf("Img Cleanup: removing image %s", id)
+				err := client.RemoveImage(id)
+				if err != nil {
+					log.Printf("Img Cleanup: %s", err)
+				}
+				counter += 1
+			}
+		}
+		log.Printf("Img Cleanup: %d images have been removed", counter)
+
+		// Sleep again
+		log.Printf("Img Cleanup: next cleanup will be start in %d seconds", *pImageCleanInterval)
+		time.Sleep(time.Duration(*pImageCleanInterval) * time.Second)
+	}
+}
+
+func cleanVolumes(client *docker.Client) {
+	log.Println("Vol Cleanup: starting volume cleanup ...")
+
+	// volumesMap[volPath] = weight
+	// weight = 0 ~ 99, increace on every iteration if it is not used
+	// weight = 100, remove it
+	volumesMap := make(map[string]int)
+	volumeDir1 := path.Join(*pDockerRootDir, "vfs/dir")
+	volumeDir2 := path.Join(*pDockerRootDir, "volumes")
+	for {
+		containers, err := client.ListContainers(docker.ListContainersOptions{All: true})
+		if err != nil {
+			log.Println("Vol Cleanup: cannot get container list", err)
+			time.Sleep(time.Duration(*pVolumeCleanInterval) * time.Second)
+			continue
+		} else {
+			inspect_error := false
+			for _, container := range containers {
+				containerInspect, err := client.InspectContainer(container.ID)
+				if err != nil {
+					inspect_error = true
+					log.Println("Vol Cleanup: cannot get container inspect", err)
+					break
+				}
+				for _, volPath := range containerInspect.Volumes {
+					volumesMap[volPath] = 0
+					if strings.Contains(volPath, "docker/vfs/dir") {
+						volPath2 := strings.Replace(volPath, "vfs/dir", "volumes", 1)
+						volumesMap[volPath2] = 0
+					}
+				}
+			}
+			if inspect_error {
+				time.Sleep(time.Duration(*pVolumeCleanInterval) * time.Second)
+				continue
+			}
+		}
+
+		files, err := ioutil.ReadDir(volumeDir1)
+		if err != nil {
+			log.Printf("Vol Cleanup: %s", err)
+		} else {
+			for _, f := range files {
+				volPath := path.Join(volumeDir1, f.Name())
+				weight := volumesMap[volPath]
+				volumesMap[volPath] = weight + 1
+			}
+		}
+
+		files, err = ioutil.ReadDir(volumeDir2)
+		if err != nil {
+			log.Printf("Vol Cleanup: %s", err)
+		} else {
+			for _, f := range files {
+				volPath := path.Join(volumeDir2, f.Name())
+				weight := volumesMap[volPath]
+				volumesMap[volPath] = weight + 1
+			}
+		}
+
+		// Remove the unused volumes
+		counter := 0
+		for volPath, weight := range volumesMap {
+			if weight == 100 {
+				log.Printf("Vol Cleanup: removing volume %s", volPath)
+				err := os.RemoveAll(volPath)
+				if err != nil {
+					log.Printf("Img Cleanup: %s", err)
+				}
+				delete(volumesMap, volPath)
+				counter += 1
+			}
+		}
+		log.Printf("Vol Cleanup: %d volumes have been removed", counter)
+
+		// Sleep
+		log.Printf("Vol Cleanup: next cleanup will be start in %d seconds", *pVolumeCleanInterval)
+		time.Sleep(time.Duration(*pVolumeCleanInterval) * time.Second)
+	}
+}