From 0e0ccc2ce2592b5801d4deb35d1fd569c616abe0 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Fri, 24 Jan 2020 20:50:28 -0700 Subject: [PATCH] Custom container build cleanup, fixes #2021, fixes #1855 (#2042) * Disambiguate built docker images from different projects, fixes #2021 * Allow users to add files/context to custom Dockerfile, fixes #1855 * Improve buildkite by removing -built images before tests * Add docs about customizing images --- .buildkite/test.sh | 5 ++--- .buildkite/testbot_maintenance.sh | 4 ++++ docs/users/extend/customizing-images.md | 9 ++++++++- pkg/ddevapp/config.go | 20 ++++++++++++++++++-- pkg/ddevapp/config_test.go | 4 ++-- pkg/ddevapp/templates.go | 6 ++++-- pkg/fileutil/files.go | 21 +++++++++++++++++++++ 7 files changed, 59 insertions(+), 10 deletions(-) diff --git a/.buildkite/test.sh b/.buildkite/test.sh index de4f5d4b64b..b3260fb2898 100755 --- a/.buildkite/test.sh +++ b/.buildkite/test.sh @@ -36,10 +36,9 @@ echo "--- running sanetestbot.sh" echo "--- cleaning up docker and Test directories" echo "Warning: deleting all docker containers and deleting ~/.ddev/Test*" if [ "$(docker ps -aq | wc -l)" -gt 0 ] ; then - docker rm -f $(docker ps -aq) || true + docker rm -f $(docker ps -aq) >/dev/null || true fi -docker system prune --volumes --force || true -docker rm -f $(docker ps -aq) || true +docker system prune --volumes --force >/dev/null || true # Update all images that could have changed ( docker images | awk '/drud/ {print $1":"$2 }' | xargs -L1 docker pull ) || true diff --git a/.buildkite/testbot_maintenance.sh b/.buildkite/testbot_maintenance.sh index 831f3563da5..9ea1ea9918a 100755 --- a/.buildkite/testbot_maintenance.sh +++ b/.buildkite/testbot_maintenance.sh @@ -29,3 +29,7 @@ windows) choco upgrade -y mkcert golang ;; esac + +# Remove any -built images, as we want to make sure tests do the building. +docker rmi -f $(docker images --filter "dangling=true" -q --no-trunc) >/dev/null || true +docker rmi -f $(docker images | awk '/drud.*-built/ {print $3}' ) >/dev/null || true diff --git a/docs/users/extend/customizing-images.md b/docs/users/extend/customizing-images.md index 8d4bcc20769..0ae8b23a6f9 100644 --- a/docs/users/extend/customizing-images.md +++ b/docs/users/extend/customizing-images.md @@ -23,12 +23,19 @@ For more complex requirements, you can add .ddev/web-build/Dockerfile or .ddev/d Examples of possible Dockerfiles are given in `.ddev/web-build/Dockerfile.example` and `.ddev/db-build/Dockerfile.example` (These examples are created in your project when you `ddev config` the project.) +You can use the .ddev/*-build/ directory as the Docker "context" directory as well. So for example if a file named README.txt exists in .ddev/web-build, you can use `ADD README.txt /` in the Dockerfile. + An example web image `.ddev/web-build/Dockerfile` might be: ``` ARG BASE_IMAGE=drud/ddev-webserver:20190422_blackfire_io FROM $BASE_IMAGE RUN npm install --global gulp-cli +ADD README.txt / ``` -Note that if a Dockerfile is provided, any config.yaml `webimage_extra_packages` or `dbimage_extra_packages` will be ignored. +Note that if a Dockerfile is provided, any config.yaml `webimage_extra_packages` or `dbimage_extra_packages` will be ignored. If you need to add packages as well as other custom configuration, add them to your Dockerfile with a line like + +``` +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confnew" --no-install-recommends --no-install-suggests php-yaml php7.3-ldap +``` diff --git a/pkg/ddevapp/config.go b/pkg/ddevapp/config.go index d80d453d754..8ccff1a31af 100644 --- a/pkg/ddevapp/config.go +++ b/pkg/ddevapp/config.go @@ -577,6 +577,8 @@ type composeYAMLVars struct { WebMount string WebBuildContext string DBBuildContext string + WebBuildDockerfile string + DBBuildDockerfile string SSHAgentBuildContext string OmitDB bool OmitDBA bool @@ -638,8 +640,10 @@ func (app *DdevApp) RenderComposeYAML() (string, error) { Username: username, UID: uid, GID: gid, - WebBuildContext: app.GetConfigPath(".webimageBuild"), - DBBuildContext: app.GetConfigPath(".dbimageBuild"), + WebBuildContext: app.GetConfigPath("web-build"), + DBBuildContext: app.GetConfigPath("db-build"), + WebBuildDockerfile: app.GetConfigPath(".webimageBuild/Dockerfile"), + DBBuildDockerfile: app.GetConfigPath(".dbimageBuild/Dockerfile"), } if app.NFSMountEnabled { templateVars.MountType = "volume" @@ -659,6 +663,18 @@ func (app *DdevApp) RenderComposeYAML() (string, error) { // Add web and db extra dockerfile info // If there is a user-provided Dockerfile, use that as the base and then add // our extra stuff like usernames, etc. + // The db-build and web-build directories are used for context + // so must exist. They usually do. + err = os.MkdirAll(app.GetConfigPath("db-build"), 0755) + if err != nil { + return "", err + } + + err = os.MkdirAll(app.GetConfigPath("web-build"), 0755) + if err != nil { + return "", err + } + err = WriteBuildDockerfile(app.GetConfigPath(".webimageBuild/Dockerfile"), app.GetConfigPath("web-build/Dockerfile"), app.WebImageExtraPackages) if err != nil { return "", err diff --git a/pkg/ddevapp/config_test.go b/pkg/ddevapp/config_test.go index 9ce3fe95b98..92529a9d0da 100644 --- a/pkg/ddevapp/config_test.go +++ b/pkg/ddevapp/config_test.go @@ -759,8 +759,8 @@ func TestExtraPackages(t *testing.T) { app.WebImageExtraPackages = nil app.DBImageExtraPackages = nil _ = app.WriteConfig() - _ = os.RemoveAll(app.GetConfigPath("web-build")) - _ = os.RemoveAll(app.GetConfigPath("db-build")) + _ = fileutil.RemoveContents(app.GetConfigPath("web-build")) + _ = fileutil.RemoveContents(app.GetConfigPath("db-build")) _ = app.Stop(true, false) }() diff --git a/pkg/ddevapp/templates.go b/pkg/ddevapp/templates.go index b00ae57b4f1..f91480e4d1f 100644 --- a/pkg/ddevapp/templates.go +++ b/pkg/ddevapp/templates.go @@ -10,12 +10,13 @@ services: container_name: {{ .Plugin }}-${DDEV_SITENAME}-db build: context: '{{ .DBBuildContext }}' + dockerfile: '{{ .DBBuildDockerfile }}' args: BASE_IMAGE: $DDEV_DBIMAGE username: '{{ .Username }}' uid: '{{ .UID }}' gid: '{{ .GID }}' - image: ${DDEV_DBIMAGE}-built + image: ${DDEV_DBIMAGE}-${DDEV_SITENAME}-built stop_grace_period: 60s volumes: - type: "volume" @@ -53,12 +54,13 @@ services: container_name: {{ .Plugin }}-${DDEV_SITENAME}-web build: context: '{{ .WebBuildContext }}' + dockerfile: '{{ .WebBuildDockerfile }}' args: BASE_IMAGE: $DDEV_WEBIMAGE username: '{{ .Username }}' uid: '{{ .UID }}' gid: '{{ .GID }}' - image: ${DDEV_WEBIMAGE}-built + image: ${DDEV_WEBIMAGE}-${DDEV_SITENAME}-built cap_add: - SYS_PTRACE volumes: diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index 3fde35987b2..fb16f451a9f 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -342,3 +342,24 @@ func ReplaceSimulatedLinks(path string) { util.Success("Replaced these simulated symlinks with real symlinks: %v", replacedLinks) return } + +// RemoveContents removes contents of passed directory +// From https://stackoverflow.com/questions/33450980/how-to-remove-all-contents-of-a-directory-using-golang +func RemoveContents(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(dir, name)) + if err != nil { + return err + } + } + return nil +}