From 52a24d7eb78eef7f44001abb60868706e8a92e10 Mon Sep 17 00:00:00 2001 From: Hardy Ferentschik Date: Mon, 3 Jul 2017 15:00:55 +0200 Subject: [PATCH] Issue #952 Implementing image caching using OCI based image cache - Switching to containers/image OCI tansport for image caching - Addressing cross compilation issues by making sure the right build tags are used - Replacing current ImageHandler with an OCI based ImageHandler - Making sure image caching work in conjunction with profiles (issue #1666) - Implementing image commands import, export and list - Adding commands image config [view|add|remove] for configuring the list of persistent cache images --- Gopkg.lock | 245 +++++++++-- Gopkg.toml | 82 +++- Makefile | 20 +- cmd/minishift/cmd/config/config.go | 12 +- cmd/minishift/cmd/config/view.go | 1 + cmd/minishift/cmd/image/cache_config.go | 38 ++ cmd/minishift/cmd/image/cache_config_add.go | 62 +++ .../cmd/image/cache_config_remove.go | 61 +++ cmd/minishift/cmd/image/cache_config_view.go | 50 +++ cmd/minishift/cmd/image/export.go | 105 ++++- cmd/minishift/cmd/image/export_test.go | 39 ++ cmd/minishift/cmd/image/image.go | 12 +- cmd/minishift/cmd/image/import.go | 106 +++++ cmd/minishift/cmd/image/import_test.go | 57 +++ cmd/minishift/cmd/image/list.go | 75 ++++ cmd/minishift/cmd/image/util.go | 143 +++++++ cmd/minishift/cmd/image/util_test.go | 44 ++ cmd/minishift/cmd/start.go | 70 ++- docs/source/using/image-caching.adoc | 139 +++++- pkg/minishift/docker/image/image.go | 16 +- pkg/minishift/docker/image/image_handler.go | 336 +-------------- .../docker/image/image_handler_test.go | 75 ---- .../docker/image/oci_image_handler.go | 397 ++++++++++++++++++ .../docker/image/oci_image_handler_test.go | 88 ++++ .../docker/image/testdata/index.json | 53 +++ pkg/util/os/process/sysproc_darwin.go | 2 +- pkg/util/os/process/sysproc_linux.go | 2 +- pkg/util/os/process/sysproc_windows.go | 4 +- pkg/util/progressdots/progressdots.go | 11 +- pkg/util/strings/strings.go | 11 + pkg/util/strings/strings_test.go | 18 + test/integration/features/basic.feature | 71 +--- test/integration/features/cmd-image.feature | 80 ++++ 33 files changed, 1945 insertions(+), 580 deletions(-) create mode 100644 cmd/minishift/cmd/image/cache_config.go create mode 100644 cmd/minishift/cmd/image/cache_config_add.go create mode 100644 cmd/minishift/cmd/image/cache_config_remove.go create mode 100644 cmd/minishift/cmd/image/cache_config_view.go create mode 100644 cmd/minishift/cmd/image/export_test.go create mode 100644 cmd/minishift/cmd/image/import.go create mode 100644 cmd/minishift/cmd/image/import_test.go create mode 100644 cmd/minishift/cmd/image/list.go create mode 100644 cmd/minishift/cmd/image/util.go create mode 100644 cmd/minishift/cmd/image/util_test.go delete mode 100644 pkg/minishift/docker/image/image_handler_test.go create mode 100644 pkg/minishift/docker/image/oci_image_handler.go create mode 100644 pkg/minishift/docker/image/oci_image_handler_test.go create mode 100644 pkg/minishift/docker/image/testdata/index.json create mode 100644 test/integration/features/cmd-image.feature diff --git a/Gopkg.lock b/Gopkg.lock index 163c1b284a..ca5e47a69c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,36 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + branch = "master" + name = "github.com/Azure/go-ansiterm" + packages = [".","winterm"] + revision = "d6e3b3328b783f23731bc4d058875b0371ff8109" + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" + [[projects]] name = "github.com/DATA-DOG/godog" packages = [".","colors","gherkin"] revision = "4dc98b0e2b130c3c9d06868a050320e5b310d3e7" version = "v0.7.4" +[[projects]] + name = "github.com/Microsoft/go-winio" + packages = [".","archive/tar","backuptar"] + revision = "78439966b38d69bf38227fbf57ac8a6fee70f69a" + version = "v0.4.5" + +[[projects]] + name = "github.com/Microsoft/hcsshim" + packages = ["."] + revision = "34a629f78a5d50f7de07727e41a948685c45e026" + version = "v0.6.7" + [[projects]] name = "github.com/asaskevich/govalidator" packages = ["."] @@ -19,22 +43,56 @@ revision = "b38d23b8782a487059e8fc8773e9a5b228a77cb6" version = "v3.5.0" +[[projects]] + name = "github.com/containers/image" + packages = ["copy","directory","directory/explicitfilepath","docker","docker/archive","docker/daemon","docker/policyconfiguration","docker/reference","docker/tarfile","image","internal/tmpdir","manifest","oci/archive","oci/internal","oci/layout","openshift","ostree","pkg/compression","pkg/docker/config","pkg/strslice","pkg/tlsclientconfig","signature","storage","tarball","transports","transports/alltransports","types","version"] + revision = "8b24210344a448e58be6ed53636e2eacaa30be32" + +[[projects]] + branch = "master" + name = "github.com/containers/storage" + packages = [".","drivers","drivers/aufs","drivers/btrfs","drivers/devmapper","drivers/overlay","drivers/overlayutils","drivers/quota","drivers/register","drivers/vfs","drivers/windows","drivers/zfs","pkg/archive","pkg/chrootarchive","pkg/devicemapper","pkg/directory","pkg/dmesg","pkg/fileutils","pkg/fsutils","pkg/homedir","pkg/idtools","pkg/ioutils","pkg/locker","pkg/longpath","pkg/loopback","pkg/mount","pkg/parsers","pkg/parsers/kernel","pkg/pools","pkg/promise","pkg/reexec","pkg/stringid","pkg/system","pkg/truncindex"] + revision = "138cddaf9d6b3910b18de44a017417f60bff4e66" + [[projects]] name = "github.com/cpuguy83/go-md2man" packages = ["md2man"] - revision = "bcc0a711c5e6bbe72c7cb13d81c7109b45267fd2" + revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1" + version = "v1.0.7" + +[[projects]] + name = "github.com/docker/distribution" + packages = [".","digestset","reference","registry/api/errcode","registry/api/v2","registry/client","registry/client/auth/challenge","registry/client/transport","registry/storage/cache","registry/storage/cache/memory"] + revision = "5f6282db7d65e6d72ad7c2cc66310724a57be716" [[projects]] name = "github.com/docker/docker" - packages = ["pkg/term"] - revision = "a8a31eff10544860d2188dddabdee4d727545796" - version = "v1.5.0" + packages = ["api","api/types","api/types/blkiodev","api/types/container","api/types/events","api/types/filters","api/types/image","api/types/mount","api/types/network","api/types/registry","api/types/strslice","api/types/swarm","api/types/swarm/runtime","api/types/time","api/types/versions","api/types/volume","client","pkg/homedir","pkg/idtools","pkg/ioutils","pkg/longpath","pkg/mount","pkg/system","pkg/term","pkg/term/windows","pkg/tlsconfig"] + revision = "30eb4d8cdc422b023d5f11f29a82ecb73554183b" + +[[projects]] + name = "github.com/docker/docker-credential-helpers" + packages = ["client","credentials"] + revision = "d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1" + version = "v0.6.0" + +[[projects]] + name = "github.com/docker/go-connections" + packages = ["nat","sockets","tlsconfig"] + revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" + version = "v0.3.0" [[projects]] name = "github.com/docker/go-units" packages = ["."] revision = "0bbddae09c5a5419a8c6dcdd7ff90da3d450393b" +[[projects]] + branch = "master" + name = "github.com/docker/libtrust" + packages = ["."] + revision = "aabc10ec26b754e797f9028f4589c5b7bd90dc20" + [[projects]] name = "github.com/docker/machine" packages = ["commands/mcndirs","drivers/errdriver","drivers/fakedriver","drivers/hyperv","drivers/none","drivers/virtualbox","drivers/vmwarefusion","libmachine","libmachine/auth","libmachine/cert","libmachine/check","libmachine/drivers","libmachine/drivers/plugin","libmachine/drivers/plugin/localbinary","libmachine/drivers/rpc","libmachine/engine","libmachine/host","libmachine/log","libmachine/mcndockerclient","libmachine/mcnerror","libmachine/mcnflag","libmachine/mcnutils","libmachine/persist","libmachine/provision","libmachine/provision/pkgaction","libmachine/provision/provisiontest","libmachine/provision/serviceaction","libmachine/shell","libmachine/ssh","libmachine/state","libmachine/swarm","libmachine/version","libmachine/versioncmp","version"] @@ -47,10 +105,10 @@ revision = "aacba83f36a55ac31cbb71c06547a328c0cd1604" [[projects]] - branch = "master" name = "github.com/fsnotify/fsnotify" packages = ["."] - revision = "4da3e2cfbabc9f751898f250b49f2439785783a1" + revision = "629574ca2a5df945712d3079857300b5e4da0236" + version = "v1.4.2" [[projects]] name = "github.com/gbraad/go-hvkvp" @@ -58,6 +116,18 @@ revision = "a692e030379b6de28a10b7697decdf8109899f6e" version = "0.3" +[[projects]] + name = "github.com/ghodss/yaml" + packages = ["."] + revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" + version = "v1.0.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = ["proto"] + revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" + version = "v0.5" + [[projects]] name = "github.com/golang/glog" packages = ["."] @@ -65,9 +135,10 @@ source = "https://github.com/openshift/glog.git" [[projects]] + branch = "master" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "3c84672111d91bb5ac31719e112f9f7126a0e26e" + revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" [[projects]] name = "github.com/google/go-github" @@ -75,9 +146,22 @@ revision = "30a21ee1a3839fb4a408efe331f226b73faac379" [[projects]] + branch = "master" name = "github.com/google/go-querystring" packages = ["query"] - revision = "30f7a39f4a218feb5325f3aebc60c32a572a8274" + revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a" + version = "v1.1" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "7f08801859139f86dfafd1c296e2cba9a80d292e" + version = "v1.6.0" [[projects]] name = "github.com/gorillalabs/go-powershell" @@ -85,12 +169,18 @@ revision = "3bc7a60b1df80a5766820c4a53d59668f4553f75" [[projects]] + branch = "master" name = "github.com/hashicorp/hcl" packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] - revision = "392dba7d905ed5d04a5794ba89f558b27e2ba1ca" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" + +[[projects]] + name = "github.com/imdario/mergo" + packages = ["."] + revision = "7fe0c75c13abdee74b09fcacef5ea1c6bba6a874" + version = "0.2.4" [[projects]] - branch = "master" name = "github.com/inconshreveable/go-update" packages = [".","internal/binarydist","internal/osext"] revision = "8152e7eb6ccf8679a64582a66b78519688d156ad" @@ -116,18 +206,38 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "51463bfca2576e06c62a8504b5c0f06d61312647" + revision = "be5ece7dd465ab0765a9682137865547526d1dfb" + version = "v1.7.3" [[projects]] - branch = "master" name = "github.com/mattn/go-runewidth" packages = ["."] - revision = "97311d9f7767e3d6f422ea06661bc2c7a19e8a5d" + revision = "9e777a8366cce605130a531d2cd6363d07ad7317" + version = "v0.0.2" + +[[projects]] + name = "github.com/mattn/go-shellwords" + packages = ["."] + revision = "02e3cf038dcea8290e44424da473dd12be796a8a" + version = "v1.0.3" + +[[projects]] + name = "github.com/mistifyio/go-zfs" + packages = ["."] + revision = "cdc0f941c4d0e0e94d85348285568d921891e138" + version = "v2.1.1" [[projects]] + branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "db1efb556f84b25a0a13a04aad883943538ad2e0" + revision = "06020f85339e21b2478f756a78e295255ffa4d6a" + +[[projects]] + branch = "master" + name = "github.com/mtrmac/gpgme" + packages = ["."] + revision = "b2432428689ca58c2b8e8dea9449d3295cf96fc9" [[projects]] name = "github.com/olekukonko/tablewriter" @@ -135,23 +245,47 @@ revision = "febf2d34b54a69ce7530036c7503b1c9fbfdf0bb" [[projects]] - name = "github.com/pborman/uuid" + name = "github.com/opencontainers/go-digest" packages = ["."] - revision = "1b00554d822231195d1babd97ff4a781231955c9" + revision = "279bed98673dd5bef374d3b6e4b09e2af76183bf" + version = "v1.0.0-rc1" + +[[projects]] + name = "github.com/opencontainers/image-spec" + packages = ["specs-go","specs-go/v1"] + revision = "d60099175f88c47cd379c4738d158884749ed235" + version = "v1.0.1" + +[[projects]] + name = "github.com/opencontainers/runc" + packages = ["libcontainer/system","libcontainer/user"] + revision = "baf6536d6259209c3edfa2b22237af82942d3dfa" + version = "v0.1.1" + +[[projects]] + name = "github.com/opencontainers/selinux" + packages = ["go-selinux","go-selinux/label"] + revision = "ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d" + version = "v1.0.0-rc1" + +[[projects]] + branch = "master" + name = "github.com/ostreedev/ostree-go" + packages = ["pkg/glibobject","pkg/otbuiltin"] + revision = "cb6250d5a6a240b509609915842f763fd87b819d" [[projects]] - name = "github.com/pelletier/go-buffruneio" + name = "github.com/pborman/uuid" packages = ["."] - revision = "c37440a7cf42ac63b919c752ca73a85067e05992" - version = "v0.2.0" + revision = "1b00554d822231195d1babd97ff4a781231955c9" [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] - revision = "048765b4491bcff26505dfbb8a7b920133a19fd2" + revision = "16398bac157da96aa88f98a2df640c7f32af1da2" + version = "v1.0.1" [[projects]] - branch = "master" name = "github.com/pkg/browser" packages = ["."] revision = "c90ca0c84f15f81c982e32665bffd8d7aac8f097" @@ -162,20 +296,35 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" +[[projects]] + branch = "master" + name = "github.com/pquerna/ffjson" + packages = ["fflib/v1","fflib/v1/internal"] + revision = "d49c2bc1aa135aad0c6f4fc2056623ec78f5d5ac" + [[projects]] name = "github.com/russross/blackfriday" packages = ["."] - revision = "0ba0f2b6ed7c475a92e4df8641825cb7a11d1fa3" + revision = "4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c" + version = "v1.5" [[projects]] + branch = "master" name = "github.com/samalba/dockerclient" packages = ["."] - revision = "f661dd4754aa5c52da85d04b5871ee0e11f4b59c" + revision = "a3036261847103270e9f732509f43b5f98710ace" + +[[projects]] + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e" + version = "v1.0.3" [[projects]] name = "github.com/spf13/afero" packages = [".","mem"] - revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" + revision = "8d919cbe7e2627e417f3e45c3c0e489a5b7e2536" + version = "v1.0.0" [[projects]] name = "github.com/spf13/cast" @@ -189,30 +338,43 @@ revision = "e606913c4ee45fec232e67e70105fb6c866b95d9" [[projects]] + branch = "master" name = "github.com/spf13/jwalterweatherman" packages = ["."] - revision = "0efa5202c04663c757d84f90f5219c1250baf94f" + revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b" [[projects]] name = "github.com/spf13/pflag" packages = ["."] revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" [[projects]] name = "github.com/spf13/viper" packages = ["."] revision = "382f87b929b84ce13e9c8a375a4b217f224e6c65" +[[projects]] + name = "github.com/tchap/go-patricia" + packages = ["patricia"] + revision = "666120de432aea38ab06bd5c818f04f4129882c9" + version = "v2.2.6" + +[[projects]] + name = "github.com/vbatts/tar-split" + packages = ["archive/tar","tar/asm","tar/storage"] + revision = "38ec4ddb06dedbea0a895c4848b248eb38af221b" + version = "v0.10.2" + [[projects]] name = "golang.org/x/crypto" - packages = ["curve25519","ssh","ssh/terminal"] - revision = "beef0f4390813b96e8e68fd78570396d0f4751fc" + packages = ["cast5","curve25519","ed25519","ed25519/internal/edwards25519","openpgp","openpgp/armor","openpgp/elgamal","openpgp/errors","openpgp/packet","openpgp/s2k","ssh","ssh/terminal"] + revision = "453249f01cfeb54c3d549ddb75ff152ca243f9d8" [[projects]] + branch = "master" name = "golang.org/x/net" - packages = ["context"] - revision = "4f2fc6c1e69d41baf187332ee08fbd2b296f21ed" + packages = ["context","context/ctxhttp","http2","http2/hpack","idna","lex/httplex","proxy"] + revision = "894f8ed5849b15b810ae41e9590a0d05395bba27" [[projects]] name = "golang.org/x/oauth2" @@ -220,19 +382,22 @@ revision = "442624c9ec9243441e83b374a9e22ac549b5c51d" [[projects]] + branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows/registry"] - revision = "d9157a9621b69ad1d8d77a1933590c416593f24f" + packages = ["unix","windows","windows/registry"] + revision = "1006bb3484c92b19a5b6612452e038b554fadb9c" [[projects]] + branch = "master" name = "golang.org/x/text" - packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] - revision = "19e51611da83d6be54ddafce4a4af510cb3e9ea4" + packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] + revision = "572a2b141f625f4360cf42a41a43622067e0510b" [[projects]] name = "google.golang.org/appengine" packages = ["internal","internal/base","internal/datastore","internal/log","internal/remote_api","internal/urlfetch","urlfetch"] - revision = "6a436539be38c296a8075a871cc536686b458371" + revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" + version = "v1.0.0" [[projects]] name = "gopkg.in/cheggaaa/pb.v1" @@ -243,11 +408,17 @@ [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b" + revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a" + +[[projects]] + name = "k8s.io/client-go" + packages = ["util/homedir"] + revision = "2ae454230481a7cb5544325e12ad7658ecccd19b" + version = "v5.0.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "02709f3b49214e62f04a8799f9997a30a8663effb4d033e9c2ea4896ba532daa" + inputs-digest = "f7e2affeedbffeaf8971abb4fcf9adeea3b4f8471eaaf2523aa9f0ba82ec3cd8" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 478ed3c284..04de08472f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -20,6 +20,8 @@ # name = "github.com/x/y" # version = "2.4.0" +# We don't want to end up with an old logrus import, only github.com/sirupsen/logrus should be used +ignored = ["github.com/Sirupsen/logrus"] [[constraint]] name = "github.com/DATA-DOG/godog" @@ -35,6 +37,7 @@ [[constraint]] name = "github.com/docker/go-units" + revision = "0bbddae09c5a5419a8c6dcdd7ff90da3d450393b" [[constraint]] name = "github.com/docker/machine" @@ -42,6 +45,7 @@ [[constraint]] name = "github.com/elazarl/goproxy" + revision = "aacba83f36a55ac31cbb71c06547a328c0cd1604" [[constraint]] name = "github.com/gbraad/go-hvkvp" @@ -62,42 +66,106 @@ [[constraint]] name = "github.com/inconshreveable/go-update" + revision = "8152e7eb6ccf8679a64582a66b78519688d156ad" [[constraint]] name = "github.com/kardianos/osext" [[constraint]] name = "github.com/olekukonko/tablewriter" - -[[constraint]] - name = "github.com/pborman/uuid" + revision = "febf2d34b54a69ce7530036c7503b1c9fbfdf0bb" [[constraint]] name = "github.com/pkg/browser" + revision = "c90ca0c84f15f81c982e32665bffd8d7aac8f097" [[constraint]] name = "github.com/pkg/errors" - version = "0.8.0" + version = "=0.8.0" [[constraint]] name = "github.com/spf13/cobra" + revision = "e606913c4ee45fec232e67e70105fb6c866b95d9" [[constraint]] name = "github.com/spf13/pflag" + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" [[constraint]] name = "github.com/spf13/viper" revision = "382f87b929b84ce13e9c8a375a4b217f224e6c65" -[[constraint]] - name = "golang.org/x/crypto" - [[constraint]] name = "golang.org/x/oauth2" + revision = "442624c9ec9243441e83b374a9e22ac549b5c51d" [[constraint]] name = "gopkg.in/cheggaaa/pb.v1" version = "=1.0.13" +# Get all dependencies for containers/image which is used for OCI image chaching +[[constraint]] + name = "github.com/containers/image" + revision = "8b24210344a448e58be6ed53636e2eacaa30be32" + +[[constraint]] + name = "github.com/opencontainers/runc" + revision = "6b1d0e76f239ffb435445e5ae316d2676c07c6e3" + +[[constraint]] + name = "github.com/docker/go-connections" + revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" + +# Use override for docker/docker and docker/distribution to ensure we get the specfied version. +# In particular we need a version which uses the altest logrus +[[override]] + name = "github.com/docker/docker" + revision = "30eb4d8cdc422b023d5f11f29a82ecb73554183b" + +[[override]] + name = "github.com/docker/distribution" + revision = "5f6282db7d65e6d72ad7c2cc66310724a57be716" + +[[constraint]] + name = "github.com/opencontainers/image-spec" + version = "=v1.0.0" + [[constraint]] name = "gopkg.in/yaml.v2" + revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a" + +[[constraint]] + name = "github.com/pborman/uuid" + revision = "1b00554d822231195d1babd97ff4a781231955c9" + +[[constraint]] + name = "golang.org/x/crypto" + revision = "453249f01cfeb54c3d549ddb75ff152ca243f9d8" + +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "=v1.0.0" + +[[constraint]] + name = "github.com/gorilla/mux" + revision = "94e7d24fd285520f3d12ae998f7fdd6b5393d453" + +[[constraint]] + name = "github.com/docker/libtrust" + revision = "aabc10ec26b754e797f9028f4589c5b7bd90dc20" + +[[constraint]] + name = "github.com/ghodss/yaml" + revision = "04f313413ffd65ce25f2541bfd2b2ceec5c0908c" + +[[constraint]] + name = "github.com/vbatts/tar-split" + version = "=v0.10.2" + +[[constraint]] + name = "golang.org/x/sys" + revision = "43e60d72a8e2bd92ee98319ba9a384a0e9837c08" + +[[constraint]] + name = "github.com/opencontainers/go-digest" + revision = "aa2ec055abd10d26d539eb630a92241b781ce4bc" diff --git a/Makefile b/Makefile index d0196a2c07..c83be56f9f 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,8 @@ VERSION_VARIABLES := -X $(REPOPATH)/pkg/version.minishiftVersion=$(MINISHIFT_VER -X $(REPOPATH)/pkg/version.openshiftVersion=$(OPENSHIFT_VERSION) \ -X $(REPOPATH)/pkg/version.commitSha=$(COMMIT_SHA) LDFLAGS := $(VERSION_VARIABLES) -s -w -extldflags '-static' +# Build tags atm mainly required to compile containers/image from which we only need OCI and Docker daemon transport. See issue #952 +BUILD_TAGS=containers_image_openpgp containers_image_storage_stub containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay # Setup for go-bindata to include binary assets ADDON_ASSETS = $(CURDIR)/addons @@ -78,7 +80,7 @@ __check_defined = \ .PHONY: $(GOPATH)/bin/minishift$(IS_EXE) $(GOPATH)/bin/minishift$(IS_EXE): $(ADDON_ASSET_FILE) vendor - go install -pkgdir=$(ADDON_BINDATA_DIR) -ldflags="$(VERSION_VARIABLES)" ./cmd/minishift + go install -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) -ldflags="$(VERSION_VARIABLES)" ./cmd/minishift vendor: dep ensure -v @@ -89,14 +91,14 @@ $(ADDON_ASSET_FILE): $(GOPATH)/bin/go-bindata $(BUILD_DIR)/$(GOOS)-$(GOARCH): mkdir -p $(BUILD_DIR)/$(GOOS)-$(GOARCH) -$(BUILD_DIR)/darwin-amd64/minishift: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH) - CGO_ENABLED=0 GOARCH=amd64 GOOS=darwin go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/darwin-amd64/minishift ./cmd/minishift +$(BUILD_DIR)/darwin-amd64/minishift: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH) + CGO_ENABLED=0 GOARCH=amd64 GOOS=darwin go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/darwin-amd64/minishift ./cmd/minishift -$(BUILD_DIR)/linux-amd64/minishift: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH) - CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/linux-amd64/minishift ./cmd/minishift +$(BUILD_DIR)/linux-amd64/minishift: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH) + CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/linux-amd64/minishift ./cmd/minishift -$(BUILD_DIR)/windows-amd64/minishift.exe: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH) - CGO_ENABLED=0 GOARCH=amd64 GOOS=windows go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/windows-amd64/minishift.exe ./cmd/minishift +$(BUILD_DIR)/windows-amd64/minishift.exe: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH) + CGO_ENABLED=0 GOARCH=amd64 GOOS=windows go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/windows-amd64/minishift.exe ./cmd/minishift $(GOPATH)/bin/gh-release: go get -u github.com/progrium/gh-release/... @@ -137,7 +139,7 @@ link_check_docs: gen_docs $(DOCS_SYNOPISIS_DIR)/*.md: vendor $(ADDON_ASSET_FILE) @# https://github.com/golang/go/issues/15038#issuecomment-207631885 ( CGO_ENABLED=0 ) - DOCS_SYNOPISIS_DIR=$(DOCS_SYNOPISIS_DIR) CGO_ENABLED=0 go run -ldflags="$(LDFLAGS)" -tags gendocs gen_help_text.go + DOCS_SYNOPISIS_DIR=$(DOCS_SYNOPISIS_DIR) CGO_ENABLED=0 go run -tags "$(BUILD_TAGS) gendocs" -ldflags="$(LDFLAGS)" gen_help_text.go .PHONY: synopsis_docs synopsis_docs: $(DOCS_SYNOPISIS_DIR)/*.md @@ -193,7 +195,7 @@ clean: .PHONY: test test: vendor $(ADDON_ASSET_FILE) - @go test -ldflags="$(VERSION_VARIABLES)" -v $(shell $(PACKAGES)) + @go test -tags "$(BUILD_TAGS)" -ldflags="$(VERSION_VARIABLES)" -v $(shell $(PACKAGES)) .PHONY: integration integration: GODOG_OPTS = --tags=basic diff --git a/cmd/minishift/cmd/config/config.go b/cmd/minishift/cmd/config/config.go index a0b27ea9a5..72fb453414 100644 --- a/cmd/minishift/cmd/config/config.go +++ b/cmd/minishift/cmd/config/config.go @@ -95,9 +95,11 @@ var ( HostFoldersMountPath = createConfigSetting("hostfolders-mountpath", SetString, nil, nil, true) HostFoldersAutoMount = createConfigSetting("hostfolders-automount", SetBool, nil, nil, true) + // Image caching ImageCaching = createConfigSetting("image-caching", SetBool, nil, nil, true) + CacheImages = createConfigSetting("cache-images", SetSlice, nil, nil, true) - // Preflight checks (before start) + // Pre-flight checks (before start) SkipCheckKVMDriver = createConfigSetting("skip-check-kvm-driver", SetBool, nil, nil, true) WarnCheckKVMDriver = createConfigSetting("warn-check-kvm-driver", SetBool, nil, nil, true) SkipCheckXHyveDriver = createConfigSetting("skip-check-xhyve-driver", SetBool, nil, nil, true) @@ -108,7 +110,8 @@ var ( WarnCheckIsoUrl = createConfigSetting("warn-check-iso-url", SetBool, nil, nil, true) SkipCheckVMDriver = createConfigSetting("skip-check-vm-driver", SetBool, nil, nil, true) WarnCheckVMDriver = createConfigSetting("warn-check-vm-driver", SetBool, nil, nil, true) - // Preflight checks (after start) + + // Pre-flight checks (after start) SkipInstanceIP = createConfigSetting("skip-check-instance-ip", SetBool, nil, nil, true) WarnInstanceIP = createConfigSetting("warn-check-instance-ip", SetBool, nil, nil, true) SkipCheckNetworkHost = createConfigSetting("skip-check-network-host", SetBool, nil, nil, true) @@ -121,7 +124,8 @@ var ( WarnCheckStorageMount = createConfigSetting("warn-check-storage-mount", SetBool, nil, nil, true) SkipCheckStorageUsage = createConfigSetting("skip-check-storage-usage", SetBool, nil, nil, true) WarnCheckStorageUsage = createConfigSetting("warn-check-storage-usage", SetBool, nil, nil, true) - // Preflight values + + // Pre-flight values CheckNetworkHttpHost = createConfigSetting("check-network-http-host", SetString, nil, nil, true) CheckNetworkPingHost = createConfigSetting("check-network-ping-host", SetString, nil, nil, true) @@ -130,7 +134,7 @@ var ( IPAddress = createConfigSetting("network-ipaddress", SetString, []setFn{validations.IsValidIPv4Address}, nil, true) Netmask = createConfigSetting("network-netmask", SetString, []setFn{validations.IsValidNetmask}, nil, true) Gateway = createConfigSetting("network-gateway", SetString, []setFn{validations.IsValidIPv4Address}, nil, true) - Nameserver = createConfigSetting("network-nameserver", SetString, []setFn{validations.IsValidIPv4Address}, nil, true) + NameServer = createConfigSetting("network-nameserver", SetString, []setFn{validations.IsValidIPv4Address}, nil, true) ) func createConfigSetting(name string, set func(MinishiftConfig, string, string) error, validations []setFn, callbacks []setFn, isApply bool) *Setting { diff --git a/cmd/minishift/cmd/config/view.go b/cmd/minishift/cmd/config/view.go index ae5901ac4b..5c94ac489c 100644 --- a/cmd/minishift/cmd/config/view.go +++ b/cmd/minishift/cmd/config/view.go @@ -59,6 +59,7 @@ var configViewCmd = &cobra.Command{ func init() { excludedConfigKeys["addons"] = true + excludedConfigKeys["cache-images"] = true configViewCmd.Flags().StringVar(&configViewFormat, "format", DefaultConfigViewFormat, `Go template format to apply to the configuration file. For more information about Go templates, see: https://golang.org/pkg/text/template/ For the list of configurable variables for the template, see the struct values section of ConfigViewTemplate at: https://godoc.org/github.com/minishift/minishift/cmd/minishift/cmd/config#ConfigViewTemplate`) diff --git a/cmd/minishift/cmd/image/cache_config.go b/cmd/minishift/cmd/image/cache_config.go new file mode 100644 index 0000000000..c5d9685922 --- /dev/null +++ b/cmd/minishift/cmd/image/cache_config.go @@ -0,0 +1,38 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "github.com/spf13/cobra" +) + +const ( + noImageSpecified = "You need to specify one or more images." +) + +var ImageCacheConfigCmd = &cobra.Command{ + Use: "cache-config SUBCOMMAND [flags]", + Short: "Controls the list of cached images which are implicitly imported and exported.", + Long: "Controls the list of cached images which are implicitly imported and exported.", + Run: func(cmd *cobra.Command, args []string) { + cmd.Help() + }, +} + +func init() { + ImageCmd.AddCommand(ImageCacheConfigCmd) +} diff --git a/cmd/minishift/cmd/image/cache_config_add.go b/cmd/minishift/cmd/image/cache_config_add.go new file mode 100644 index 0000000000..1da95d94e3 --- /dev/null +++ b/cmd/minishift/cmd/image/cache_config_add.go @@ -0,0 +1,62 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "github.com/spf13/cobra" + + "fmt" + "github.com/minishift/minishift/cmd/minishift/cmd/config" + "github.com/minishift/minishift/pkg/util/os/atexit" + "github.com/minishift/minishift/pkg/util/strings" +) + +var ( + addConfiguredImageCmd = &cobra.Command{ + Use: "add [image ...]", + Short: "Adds the specified images to the list of configured images for import and export.", + Long: "Adds the specified images to the list of configured images for import and export.", + Run: addConfiguredImage, + } +) + +func addConfiguredImage(cmd *cobra.Command, args []string) { + if len(args) == 0 { + atexit.ExitWithMessage(1, noImageSpecified) + } + + normalizedImageNames, err := normalizeImageNames(args) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Invalid image name: %v", err)) + } + + minishiftConfig := getMinishiftConfig() + cacheImages := getConfiguredCachedImages(minishiftConfig) + + for _, image := range normalizedImageNames { + if !strings.Contains(cacheImages, image) { + cacheImages = append(cacheImages, image) + } + } + + minishiftConfig[config.CacheImages.Name] = cacheImages + config.WriteConfig(minishiftConfig) +} + +func init() { + ImageCacheConfigCmd.AddCommand(addConfiguredImageCmd) +} diff --git a/cmd/minishift/cmd/image/cache_config_remove.go b/cmd/minishift/cmd/image/cache_config_remove.go new file mode 100644 index 0000000000..9075616aa1 --- /dev/null +++ b/cmd/minishift/cmd/image/cache_config_remove.go @@ -0,0 +1,61 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "github.com/spf13/cobra" + + "fmt" + "github.com/minishift/minishift/cmd/minishift/cmd/config" + "github.com/minishift/minishift/pkg/util/os/atexit" + "github.com/minishift/minishift/pkg/util/strings" +) + +var ( + removeConfiguredImageCmd = &cobra.Command{ + Use: "remove [image ...]", + Aliases: []string{"rm"}, + Short: "Removes the specified images from the list of configured images for import and export.", + Long: "Removes the specified images from the list of configured images for import and export.", + Run: removeConfiguredImage, + } +) + +func removeConfiguredImage(cmd *cobra.Command, args []string) { + if len(args) == 0 { + atexit.ExitWithMessage(1, noImageSpecified) + } + + normalizedImageNames, err := normalizeImageNames(args) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Invalid image name: %v", err)) + } + + minishiftConfig := getMinishiftConfig() + cacheImages := getConfiguredCachedImages(minishiftConfig) + + for _, image := range normalizedImageNames { + cacheImages = strings.Remove(cacheImages, image) + } + + minishiftConfig[config.CacheImages.Name] = cacheImages + config.WriteConfig(minishiftConfig) +} + +func init() { + ImageCacheConfigCmd.AddCommand(removeConfiguredImageCmd) +} diff --git a/cmd/minishift/cmd/image/cache_config_view.go b/cmd/minishift/cmd/image/cache_config_view.go new file mode 100644 index 0000000000..6e2c21a595 --- /dev/null +++ b/cmd/minishift/cmd/image/cache_config_view.go @@ -0,0 +1,50 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "github.com/spf13/cobra" + + "fmt" + "sort" + "strings" +) + +var ( + listConfiguredImagesCmd = &cobra.Command{ + Use: "view", + Short: "Displays the configured list of images for import and export.", + Long: "Displays the configured list of images for import and export.", + Run: viewConfiguredImages, + } +) + +func viewConfiguredImages(cmd *cobra.Command, args []string) { + minishiftConfig := getMinishiftConfig() + cacheImages := getConfiguredCachedImages(minishiftConfig) + + if len(cacheImages) == 0 { + return + } + + sort.Strings(cacheImages) + fmt.Println(strings.Join(cacheImages, "\n")) +} + +func init() { + ImageCacheConfigCmd.AddCommand(listConfiguredImagesCmd) +} diff --git a/cmd/minishift/cmd/image/export.go b/cmd/minishift/cmd/image/export.go index f8cd25722e..3ed8299e77 100644 --- a/cmd/minishift/cmd/image/export.go +++ b/cmd/minishift/cmd/image/export.go @@ -18,35 +18,44 @@ package image import ( "fmt" + "os" + "path/filepath" + "time" + "github.com/docker/machine/libmachine" + "github.com/minishift/minishift/cmd/minishift/cmd/config" "github.com/minishift/minishift/cmd/minishift/cmd/util" + "github.com/minishift/minishift/pkg/minikube/cluster" "github.com/minishift/minishift/pkg/minikube/constants" "github.com/minishift/minishift/pkg/minishift/docker/image" "github.com/minishift/minishift/pkg/util/os/atexit" "github.com/spf13/cobra" - "os" - "path/filepath" - "time" + "github.com/spf13/viper" + "io" ) -var imageExportCmd = &cobra.Command{ - Use: "export [image ...]", - Short: "Exports the specified container images (experimental).", - Long: "Exports the specified container images (experimental).", - Run: exportImage, -} +var ( + logToFile bool + exportAll bool -func exportImage(cmd *cobra.Command, args []string) { - logFile := createLogFile() - defer logFile.Close() + imageExportCmd = &cobra.Command{ + Use: "export [image ...]", + Short: "Exports the specified container images.", + Long: "Exports the specified container images.", + Run: exportImage, + } + + CacheDir = []string{"cache", "images"} +) + +const ( + noDockerDaemonImages = "There are currently no images in the Docker daemon which can be exported." +) +func exportImage(cmd *cobra.Command, args []string) { api := libmachine.NewClient(constants.Minipath, constants.MakeMiniPath("certs")) defer api.Close() - if len(args) < 1 { - atexit.ExitWithMessage(0, "You must specify at least one container image.") - } - util.ExitIfUndefined(api, constants.MachineName) host, err := api.Load(constants.MachineName) @@ -56,26 +65,74 @@ func exportImage(cmd *cobra.Command, args []string) { util.ExitIfNotRunning(host.Driver, constants.MachineName) - handler, err := image.NewDockerImageHandler(host.Driver) + envMap, err := cluster.GetHostDockerEnv(api) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error determining Docker daemon settings: %v", err)) + } + + var out io.Writer + if logToFile { + logFile := createLogFile() + defer logFile.Close() + out = logFile + } else { + out = os.Stdout + } + + images := imagesToExport(api, args) + + handler, err := image.NewOciImageHandler(host.Driver, envMap) if err != nil { atexit.ExitWithMessage(1, fmt.Sprintf("Cannot create the image handler: %v", err)) } + normalizedImageNames, err := normalizeImageNames(images) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Invalid image name: %v", err)) + } + imageCacheConfig := &image.ImageCacheConfig{ - HostCacheDir: constants.MakeMiniPath("cache", "images"), - CachedImages: args, - Out: logFile, - ImageMissStrategy: image.PULL, + HostCacheDir: constants.MakeMiniPath(CacheDir...), + CachedImages: normalizedImageNames, + Out: out, + ImageMissStrategy: image.Pull, } + err = handler.ExportImages(imageCacheConfig) if err != nil { - atexit.ExitWithMessage(1, fmt.Sprintf("Failed to export the container images: %v", err)) + msg := fmt.Sprintf("Failed to export the container images: %v", err) + if logToFile { + fmt.Fprint(out, msg) + } + atexit.ExitWithMessage(1, msg) + } +} + +func imagesToExport(api *libmachine.Client, args []string) []string { + var images []string + if exportAll { + images = getDockerDaemonImages(api) + } else if len(args) == 0 { + images = viper.GetStringSlice(config.CacheImages.Name) + } else { + images = args } + + if len(images) == 0 { + msg := noCachedImagesSpecified + if importAll { + msg = noDockerDaemonImages + } + atexit.ExitWithMessage(0, msg) + } + + return images + } func createLogFile() *os.File { now := time.Now() - timeStamp := now.Format("2017-01-02-1504-00") + timeStamp := now.Format("2006-01-02-1504-05") // reference time Mon Jan 2 15:04:05 -0700 MST 2006 logFilePath := filepath.Join(constants.MakeMiniPath("logs"), fmt.Sprintf("image-export-%s.log", timeStamp)) logFile, err := os.Create(logFilePath) if err != nil { @@ -86,5 +143,7 @@ func createLogFile() *os.File { } func init() { + imageExportCmd.Flags().BoolVar(&exportAll, "all", false, "Exports all images currently available in the Docker daemon.") + imageExportCmd.Flags().BoolVar(&logToFile, "log-to-file", false, "Logs export progress to file instead of standard out.") ImageCmd.AddCommand(imageExportCmd) } diff --git a/cmd/minishift/cmd/image/export_test.go b/cmd/minishift/cmd/image/export_test.go new file mode 100644 index 0000000000..5301b230fc --- /dev/null +++ b/cmd/minishift/cmd/image/export_test.go @@ -0,0 +1,39 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "testing" + + "github.com/minishift/minishift/pkg/testing/cli" + "github.com/minishift/minishift/pkg/util/os/atexit" +) + +func Test_no_images_to_export(t *testing.T) { + tee := cli.CreateTee(t, true) + defer cli.TearDown("", tee) + expectedOut := noCachedImagesSpecified + + atexit.RegisterExitHandler(cli.VerifyExitCodeAndMessage(t, tee, 0, expectedOut)) + + imagesToExport(nil, nil) + + actualOut := tee.StdoutBuffer.String() + if expectedOut != actualOut { + t.Fatalf("Expected output '%s'. Got '%s'.", expectedOut, actualOut) + } +} diff --git a/cmd/minishift/cmd/image/image.go b/cmd/minishift/cmd/image/image.go index 08eddb4949..88174da31f 100644 --- a/cmd/minishift/cmd/image/image.go +++ b/cmd/minishift/cmd/image/image.go @@ -20,11 +20,15 @@ import ( "github.com/spf13/cobra" ) +const ( + noCachedImagesSpecified = "You need to either specify a list of images on the command line or configure the list of cached images via 'image config [add|remove]'." +) + var ImageCmd = &cobra.Command{ - Use: "image SUBCOMMAND [flags]", - Short: "Exports and imports container images (experimental).", - Long: "Exports and imports container images (experimental).", - Hidden: true, + Use: "image SUBCOMMAND [flags]", + Aliases: []string{"images"}, + Short: "Exports and imports container images.", + Long: "Exports and imports container images.", Run: func(cmd *cobra.Command, args []string) { cmd.Help() }, diff --git a/cmd/minishift/cmd/image/import.go b/cmd/minishift/cmd/image/import.go new file mode 100644 index 0000000000..b76f623cd7 --- /dev/null +++ b/cmd/minishift/cmd/image/import.go @@ -0,0 +1,106 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "fmt" + "os" + + "github.com/docker/machine/libmachine" + "github.com/minishift/minishift/cmd/minishift/cmd/config" + "github.com/minishift/minishift/cmd/minishift/cmd/util" + "github.com/minishift/minishift/pkg/minikube/cluster" + "github.com/minishift/minishift/pkg/minikube/constants" + "github.com/minishift/minishift/pkg/minishift/docker/image" + "github.com/minishift/minishift/pkg/util/os/atexit" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + importAll bool + + imageImportCmd = &cobra.Command{ + Use: "import [image ...]", + Short: "Imports the specified images into the Docker daemon.", + Long: "Imports the specified images into the Docker daemon.", + Run: importImage, + } +) + +const ( + noCachedImages = "There are currently no images in the local cache." +) + +func importImage(cmd *cobra.Command, args []string) { + cacheDir := constants.MakeMiniPath(CacheDir...) + var images []string + if importAll { + images = getCachedImages(cacheDir) + } else if len(args) == 0 { + images = viper.GetStringSlice(config.CacheImages.Name) + } else { + images = args + } + + if len(images) == 0 { + msg := noCachedImagesSpecified + if importAll { + msg = noCachedImages + } + atexit.ExitWithMessage(0, msg) + } + + api := libmachine.NewClient(constants.Minipath, constants.MakeMiniPath("certs")) + defer api.Close() + + util.ExitIfUndefined(api, constants.MachineName) + + host, err := api.Load(constants.MachineName) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error creating the VM client: %v", err)) + } + + util.ExitIfNotRunning(host.Driver, constants.MachineName) + + envMap, err := cluster.GetHostDockerEnv(api) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error determining Docker daemon settings: %v", err)) + } + + handler, err := image.NewOciImageHandler(host.Driver, envMap) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Cannot create the image handler: %v", err)) + } + + imageCacheConfig := &image.ImageCacheConfig{ + HostCacheDir: constants.MakeMiniPath(CacheDir...), + CachedImages: images, + Out: os.Stdout, + ImageMissStrategy: image.Skip, + } + + err = handler.ImportImages(imageCacheConfig) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Failed to import the container images: %v", err)) + } +} + +func init() { + imageImportCmd.Flags().BoolVar(&importAll, "all", false, "Imports all images available in the local image cache.") + ImageCmd.AddCommand(imageImportCmd) +} diff --git a/cmd/minishift/cmd/image/import_test.go b/cmd/minishift/cmd/image/import_test.go new file mode 100644 index 0000000000..dd1cc36e2e --- /dev/null +++ b/cmd/minishift/cmd/image/import_test.go @@ -0,0 +1,57 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "testing" + + "github.com/minishift/minishift/pkg/testing/cli" + "github.com/minishift/minishift/pkg/util/os/atexit" +) + +func Test_no_images_to_import(t *testing.T) { + tee := cli.CreateTee(t, true) + defer cli.TearDown("", tee) + expectedOut := noCachedImagesSpecified + + atexit.RegisterExitHandler(cli.VerifyExitCodeAndMessage(t, tee, 0, expectedOut)) + + importImage(nil, nil) + + actualOut := tee.StdoutBuffer.String() + if expectedOut != actualOut { + t.Fatalf("Expected output '%s'. Got '%s'.", expectedOut, actualOut) + } +} + +func Test_no_images_cached(t *testing.T) { + tmpMinishiftHomeDir := cli.SetupTmpMinishiftHome(t) + tee := cli.CreateTee(t, true) + defer cli.TearDown(tmpMinishiftHomeDir, tee) + importAll = true + + expectedOut := noCachedImages + + atexit.RegisterExitHandler(cli.VerifyExitCodeAndMessage(t, tee, 0, expectedOut)) + + importImage(nil, nil) + + actualOut := tee.StdoutBuffer.String() + if expectedOut != actualOut { + t.Fatalf("Expected output '%s'. Got '%s'.", expectedOut, actualOut) + } +} diff --git a/cmd/minishift/cmd/image/list.go b/cmd/minishift/cmd/image/list.go new file mode 100644 index 0000000000..ec1026a88e --- /dev/null +++ b/cmd/minishift/cmd/image/list.go @@ -0,0 +1,75 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "fmt" + "github.com/docker/machine/libmachine" + "github.com/minishift/minishift/pkg/minikube/constants" + "github.com/spf13/cobra" +) + +var ( + dockerDaemonImages bool + + imageCacheListCmd = &cobra.Command{ + Use: "list ", + Short: "Displays the locally cached images.", + Long: "Displays the locally cached images.", + Run: listImages, + } +) + +func listImages(cmd *cobra.Command, args []string) { + api := libmachine.NewClient(constants.Minipath, constants.MakeMiniPath("certs")) + defer api.Close() + + if dockerDaemonImages { + listDockerDaemonImages(api) + } else { + listCachedImages() + } +} + +func listCachedImages() { + cacheDir := constants.MakeMiniPath(CacheDir...) + cachedImages := getCachedImages(cacheDir) + if len(cachedImages) > 0 { + printImageList(cachedImages) + } +} + +func listDockerDaemonImages(api *libmachine.Client) { + images := getDockerDaemonImages(api) + if len(images) == 0 { + fmt.Println(fmt.Sprintf("There are no images available in the Docker daemon of Minishift instance '%s'", constants.ProfileName)) + } else { + printImageList(images) + + } +} + +func printImageList(images []string) { + for _, i := range images { + fmt.Println(i) + } +} + +func init() { + imageCacheListCmd.Flags().BoolVar(&dockerDaemonImages, "vm", false, "Prints the available images in the Docker daemon.") + ImageCmd.AddCommand(imageCacheListCmd) +} diff --git a/cmd/minishift/cmd/image/util.go b/cmd/minishift/cmd/image/util.go new file mode 100644 index 0000000000..2a45fdea03 --- /dev/null +++ b/cmd/minishift/cmd/image/util.go @@ -0,0 +1,143 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "fmt" + "github.com/containers/image/docker/reference" + "github.com/docker/machine/libmachine" + "github.com/minishift/minishift/cmd/minishift/cmd/config" + "github.com/minishift/minishift/cmd/minishift/cmd/util" + "github.com/minishift/minishift/pkg/minikube/cluster" + "github.com/minishift/minishift/pkg/minikube/constants" + "github.com/minishift/minishift/pkg/minishift/docker/image" + "github.com/minishift/minishift/pkg/util/os/atexit" + "os" + "sort" +) + +func getCachedImages(cacheDir string) []string { + api := libmachine.NewClient(constants.Minipath, constants.MakeMiniPath("certs")) + defer api.Close() + + handler, err := image.NewLocalOnlyOciImageHandler() + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Cannot create the image handler: %v", err)) + } + + imageCacheConfig := &image.ImageCacheConfig{ + HostCacheDir: cacheDir, + Out: os.Stdout, + ImageMissStrategy: image.Skip, + } + + images := handler.GetCachedImages(imageCacheConfig) + return sortImageNames(images) +} + +func getDockerDaemonImages(api *libmachine.Client) []string { + util.ExitIfUndefined(api, constants.MachineName) + + host, err := api.Load(constants.MachineName) + if err != nil { + atexit.ExitWithMessage(1, err.Error()) + } + + util.ExitIfNotRunning(host.Driver, constants.MachineName) + + envMap, err := cluster.GetHostDockerEnv(api) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error determining Docker daemon settings: %v", err)) + } + + handler, err := image.NewOciImageHandler(host.Driver, envMap) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Cannot create the image handler: %v", err)) + } + + images, err := handler.GetDockerImages() + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error retrieving image list from Docker daemon: %v", err)) + } + return sortImageNames(images) +} + +func normalizeImageNames(images []string) ([]string, error) { + normalizedImageNames := []string{} + for _, image := range images { + normalizedName, err := normalizeImageName(image) + if err != nil { + return nil, err + } + normalizedImageNames = append(normalizedImageNames, normalizedName) + } + return normalizedImageNames, nil +} + +func normalizeImageName(name string) (string, error) { + ref, err := reference.Parse(name) + if err != nil { + return "", err + } + + _, ok := ref.(reference.Tagged) + if ok { + return ref.String(), nil + } + + ref, err = reference.WithTag(ref.(reference.Named), "latest") + if err != nil { + return "", err + } + + return ref.String(), nil +} + +func sortImageNames(images map[string]bool) []string { + var sortedImageList []string + for i := range images { + sortedImageList = append(sortedImageList, i) + } + sort.Strings(sortedImageList) + return sortedImageList +} + +func toStringSlice(interfaceSlice []interface{}) []string { + var slice []string + for _, s := range interfaceSlice { + slice = append(slice, s.(string)) + } + return slice +} + +func getConfiguredCachedImages(minishiftConfig config.MinishiftConfig) []string { + var cacheImages []string + if minishiftConfig[config.CacheImages.Name] == nil { + cacheImages = []string{} + } else { + cacheImages = toStringSlice(minishiftConfig[config.CacheImages.Name].([]interface{})) + } + return cacheImages +} + +func getMinishiftConfig() config.MinishiftConfig { + minishiftConfig, err := config.ReadConfig() + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Cannot read the Minishift configuration: %s", err.Error())) + } + return minishiftConfig +} diff --git a/cmd/minishift/cmd/image/util_test.go b/cmd/minishift/cmd/image/util_test.go new file mode 100644 index 0000000000..9d45580ba7 --- /dev/null +++ b/cmd/minishift/cmd/image/util_test.go @@ -0,0 +1,44 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "testing" +) + +type testData struct { + image string + normalizedImage string + err error +} + +var tests = []testData{ + {image: "alpine", normalizedImage: "alpine:latest"}, + {image: "alpine:1.24", normalizedImage: "alpine:1.24"}, +} + +func Test_normalize_image_names(t *testing.T) { + for _, test := range tests { + normalizedImage, err := normalizeImageName(test.image) + if err != test.err { + t.Errorf("Expected error '%v' , got '%v'", test.err, err) + } + if normalizedImage != test.normalizedImage { + t.Errorf("Normalinzing '%s' should have returned '%s', got '%s'", test.image, normalizedImage, test.normalizedImage) + } + } +} diff --git a/cmd/minishift/cmd/start.go b/cmd/minishift/cmd/start.go index 47ad9c9964..4616d83919 100644 --- a/cmd/minishift/cmd/start.go +++ b/cmd/minishift/cmd/start.go @@ -18,7 +18,6 @@ package cmd import ( "fmt" - "os" "runtime" "strings" @@ -31,6 +30,7 @@ import ( "github.com/golang/glog" "github.com/minishift/minishift/cmd/minishift/cmd/addon" configCmd "github.com/minishift/minishift/cmd/minishift/cmd/config" + imageCmd "github.com/minishift/minishift/cmd/minishift/cmd/image" registrationUtil "github.com/minishift/minishift/cmd/minishift/cmd/registration" cmdUtil "github.com/minishift/minishift/cmd/minishift/cmd/util" "github.com/minishift/minishift/pkg/minikube/cluster" @@ -46,7 +46,6 @@ import ( "github.com/minishift/minishift/pkg/minishift/openshift" profileActions "github.com/minishift/minishift/pkg/minishift/profile" "github.com/minishift/minishift/pkg/minishift/provisioner" - "github.com/minishift/minishift/pkg/util" "github.com/minishift/minishift/pkg/util/os/atexit" "github.com/minishift/minishift/pkg/util/progressdots" @@ -55,6 +54,7 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" "github.com/spf13/viper" + "os" ) const ( @@ -148,6 +148,8 @@ For the latter see 'minishift config -h'.`, // runStart handles all command line arguments, launches the VM and provisions OpenShift func runStart(cmd *cobra.Command, args []string) { + fmt.Println(fmt.Sprintf("-- Starting profile '%s'", constants.ProfileName)) + libMachineClient := libmachine.NewClient(constants.Minipath, constants.MakeMiniPath("certs")) defer libMachineClient.Close() @@ -165,8 +167,8 @@ func runStart(cmd *cobra.Command, args []string) { proxyConfig := handleProxies() - fmt.Println(fmt.Sprintf("-- Starting profile '%s'", constants.ProfileName)) - fmt.Printf("-- Starting local OpenShift cluster") + fmt.Print("-- Starting local OpenShift cluster") + hostVm := startHost(libMachineClient) registrationUtil.RegisterHost(libMachineClient) @@ -195,7 +197,7 @@ func runStart(cmd *cobra.Command, args []string) { requestedOpenShiftVersion := viper.GetString(configCmd.OpenshiftVersion.Name) if !isRestart { - importContainerImages(hostVm, requestedOpenShiftVersion) + importContainerImages(hostVm.Driver, libMachineClient, requestedOpenShiftVersion) } ocPath := cmdUtil.CacheOc(clusterup.DetermineOcVersion(requestedOpenShiftVersion)) @@ -229,7 +231,7 @@ func runStart(cmd *cobra.Command, args []string) { if !isRestart { postClusterUp(hostVm, clusterUpConfig) - exportContainerImages(hostVm, requestedOpenShiftVersion) + exportContainerImages(hostVm.Driver, libMachineClient, requestedOpenShiftVersion) } if isRestart { err = cmdUtil.SetOcContext(minishiftConfig.AllInstancesConfig.ActiveProfile) @@ -343,7 +345,7 @@ func startHost(libMachineClient *libmachine.Client) *host.Host { IPAddress: viper.GetString(configCmd.IPAddress.Name), Netmask: viper.GetString(configCmd.Netmask.Name), Gateway: viper.GetString(configCmd.Gateway.Name), - DNS1: viper.GetString(configCmd.Nameserver.Name), + DNS1: viper.GetString(configCmd.NameServer.Name), } // Configure networking on startup only works on Hyper-V @@ -390,21 +392,37 @@ func addActiveProfileInformation() { } } -func importContainerImages(hostVm *host.Host, openShiftVersion string) { +func importContainerImages(driver drivers.Driver, api libmachine.API, openShiftVersion string) { if !viper.GetBool(configCmd.ImageCaching.Name) { return } - handler := getImageHandler(hostVm) + images := viper.GetStringSlice(configCmd.CacheImages.Name) + for _, coreImage := range image.GetOpenShiftImageNames(openShiftVersion) { + if !stringUtils.Contains(images, coreImage) { + images = append(images, coreImage) + } + } + + envMap, err := cluster.GetHostDockerEnv(api) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error determining Docker settings for image import: %v", err)) + } + + handler := getImageHandler(driver, envMap) config := &image.ImageCacheConfig{ - HostCacheDir: constants.MakeMiniPath("cache", "images"), - CachedImages: image.GetOpenShiftImageNames(openShiftVersion), + HostCacheDir: constants.MakeMiniPath(imageCmd.CacheDir...), + CachedImages: images, + Out: os.Stdout, + } + err = handler.ImportImages(config) + if err != nil { + fmt.Println(fmt.Sprintf(" WARN: Import of cached images failed. Continuing without importing images. Error: %s ", err.Error())) } - handler.ImportImages(config) } -func getImageHandler(hostVm *host.Host) image.ImageHandler { - handler, err := image.NewDockerImageHandler(hostVm.Driver) +func getImageHandler(driver drivers.Driver, envMap map[string]string) image.ImageHandler { + handler, err := image.NewOciImageHandler(driver, envMap) if err != nil { atexit.ExitWithMessage(1, fmt.Sprintf("Unable to create image handler: %v", err)) } @@ -413,22 +431,34 @@ func getImageHandler(hostVm *host.Host) image.ImageHandler { } // exportContainerImages exports the OpenShift images in a background process (by calling 'minishift image export') -func exportContainerImages(hostVm *host.Host, version string) { +func exportContainerImages(driver drivers.Driver, api libmachine.API, version string) { if !viper.GetBool(configCmd.ImageCaching.Name) { return } - handler := getImageHandler(hostVm) + images := viper.GetStringSlice(configCmd.CacheImages.Name) + for _, coreImage := range image.GetOpenShiftImageNames(version) { + if !stringUtils.Contains(images, coreImage) { + images = append(images, coreImage) + } + } + + envMap, err := cluster.GetHostDockerEnv(api) + if err != nil { + atexit.ExitWithMessage(1, fmt.Sprintf("Error determining Docker settings for image import: %v", err)) + } + + handler := getImageHandler(driver, envMap) config := &image.ImageCacheConfig{ - HostCacheDir: constants.MakeMiniPath("cache", "images"), - CachedImages: image.GetOpenShiftImageNames(version), + HostCacheDir: constants.MakeMiniPath(imageCmd.CacheDir...), + CachedImages: images, } if handler.AreImagesCached(config) { return } - exportCmd, err := image.CreateExportCommand(version) + exportCmd, err := image.CreateExportCommand(version, constants.ProfileName, images) if err != nil { atexit.ExitWithMessage(1, fmt.Sprintf("Error creating export command: %v", err)) } @@ -510,7 +540,7 @@ func initStartFlags() *flag.FlagSet { startFlagSet.String(configCmd.IPAddress.Name, "", "Specify IP address to assign to the instance (experimental - Hyper-V only)") startFlagSet.String(configCmd.Netmask.Name, "24", "Specify netmask to use for the IP address. Ignored if no IP address specified (experimental - Hyper-V only)") startFlagSet.String(configCmd.Gateway.Name, "", "Specify gateway to use for the instance. Ignored if no IP address specified (experimental - Hyper-V only)") - startFlagSet.String(configCmd.Nameserver.Name, "8.8.8.8", "Specify nameserver to use for the instance. Ignored if no IP address specified (experimental - Hyper-V only)") + startFlagSet.String(configCmd.NameServer.Name, "8.8.8.8", "Specify nameserver to use for the instance. Ignored if no IP address specified (experimental - Hyper-V only)") } if minishiftConfig.EnableExperimental { diff --git a/docs/source/using/image-caching.adoc b/docs/source/using/image-caching.adoc index 2c769146d7..7bc37eb579 100644 --- a/docs/source/using/image-caching.adoc +++ b/docs/source/using/image-caching.adoc @@ -4,39 +4,150 @@ include::variables.adoc[] :icons: :toc: macro :toc-title: -:toclevels: 1 +:toclevels: 2 toc::[] [[image-caching-overview]] == Overview -To speed up provisioning of the OpenShift cluster and to minimize network traffic, the core OpenShift images can be cached on the host. -This feature is considered experimental and needs to be explicitly enabled. +To speed up the provisioning of the OpenShift cluster and to minimize network traffic, container images can be cached on the host. +These images can then be imported into the running Docker daemon, either explicitly on request or implicitly during xref:../using/basic-usage.adoc#minishift-start-overview[`minishift start`]. +The following sections describe image caching and its configuration in more detail. -[[image-chaching-configuration]] -== Image Caching Configuration +[TIP] +==== +The format in which images are cached has changed with {project} version 1.10.0. +Prior to 1.10.0 the images were stored as tar files. +As of 1.10.0, images are stored in the link:https://github.com/opencontainers/image-spec/blob/master/spec.md[OCI image format]. + +If you used image caching prior to {project} 1.10.0, your cache will need to be recreated. +If you want to remove the obsolete pre 1.10.0 images, you can clear your cache via: +---- +$ minishift delete --clear-cache +---- +==== + +[[explicit-image-caching]] +== Explicit Image Caching + +{project} provides the `image` command together with its sub-commands to control the behavior of image caching. +To export and import images from the Docker daemon of the {project} VM, use `minishift image export` and `minishift image import`. + +[[single-images]] +=== Importing and Exporting Single Images -To enable image caching you use the xref:../command-ref/minishift_config_set#[`minishift config set`] command: +Once the {project} VM is running, images can be explicitly exported from the Docker daemon: + +---- +$ minishift image export ... +Pulling image .. OK +Exporting . OK +Pulling image .. OK +Exporting . OK +---- + +[NOTE] +==== +Images which are not available in the Docker daemon will be pulled prior to being exported to the host. +==== + +To import previously cached images, use the xref:../command-ref/minishift_image_import.adoc#[`minishift image import`] command: + +---- +$ minishift image import ... +Importing . OK +---- + +[[listing-cached-images]] +=== Listing Cached Images + +The xref:../command-ref/minishift_image_list.adoc#[`minishift image list`] command lists either the currently cached images or the images available in the {project} Docker daemon. + +To view currently cached images on the host: + +---- +$ minishift image list +openshift/origin-docker-registry:v3.6.0 +openshift/origin-haproxy-router:v3.6.0 +openshift/origin:v3.6.0 +---- + +To view images available in the Docker daemon: + +---- +$ minishift image list --vm +openshift/origin-deployer:v3.6.0 +openshift/origin-docker-registry:v3.6.0 +openshift/origin-haproxy-router:v3.6.0 +openshift/origin-pod:v3.6.0 +openshift/origin:v3.6.0 +---- + +[[persisting-image-names]] +=== Persisting Cached Image Names + +In order to avoid having to type the image names explicitly as part of the `image export` or `image import` command, you can store a list of image names for import and export in the persistent configuration. + +Use xref:../command-ref/minishift_image_cache-config_view#[`minishift image cache-config view`] to view the list of currently configured images and xref:../command-ref/minishift_image_cache-config_add#[`minishift image cache-config add`] to add images to the list: + +---- +$ minishift image cache-config view +$ minishift image cache-config add alpine:latest busybox:latest +$ minishift image cache-config view +alpine:latest +busybox:latest +---- + +To remove images from the list use xref:../command-ref/minishift_image_cache-config_remove#[`minishift image cache-config remove`]: + +---- +$ minishift image cache-config remove alpine:latest +$ minishift image cache-config view +busybox:latest +---- + +Once the image names are stored in the persistent configuration, you can run xref:../command-ref/minishift_image_export#[`minishift image export`] and xref:../command-ref/minishift_image_import#[`minishift image import`] without any arguments. + +[[all-images]] +=== Exporting and Importing All Images + +You can export and import all images using the `--all` flag. +For the export command, this means that all images currently available on the Docker daemon will be exported to the host. +For the import command, it means that all images available in the local {project} cache will be imported into the Docker daemon of the {project} VM. + +[WARNING] +==== +Exporting and importing all images can take a long time and locally cached images can take up a considerable amount of disk space. +We recommend using this feature with caution. +==== + +[[implicit-image-caching]] +== Implicit Image Caching + +Image caching can be configured to automatically occur during the creation of the {project} VM while executing `minishift start`. +To enable this feature you need to enable the `image-caching` property in the persistent configuration using the xref:../command-ref/minishift_config_set#[`minishift config set`] command: ---- $ minishift config set image-caching true ---- -Once enabled, caching occurs transparently, in a background process, the first time you use the xref:../command-ref/minishift_start#[`minishift start`] command. +Once enabled, caching occurs in a background process, the first time you use the xref:../command-ref/minishift_start#[`minishift start`] command. Once the images are cached under *_$MINISHIFT_HOME/cache/images_*, successive {project} VM creations will use these cached images. +[NOTE] +==== +Enabling implicit image caching will transparently add the required OpenShift images to the list of cached images as specified per `cache-images` configuration option. +See xref:../using/image-caching.adoc#persisting-image-names[Persisting Cached Image Names]. +==== + +[TIP] +==== Each time an image exporting background process runs, a log file is generated under *_$MINISHIFT_HOME/logs_* which can be used to verify the progress of the export. +==== You can disable the caching of the OpenShift images by setting `image-caching` to `false` or removing the setting altogether using xref:../command-ref/minishift_config_unset#[`minishift config unset`]: ---- $ minishift config unset image-caching ---- - -[NOTE] -==== -Image caching is considered experimental and its semantics and API are subject to change. -The aim is to allow caching of arbitrary images, as well as using a better format for storing the images on the host. -You can track the progress on this feature on GitHub issue link:https://github.com/minishift/minishift/issues/952[#952]. -==== diff --git a/pkg/minishift/docker/image/image.go b/pkg/minishift/docker/image/image.go index 17d87529be..c2d716e0a0 100644 --- a/pkg/minishift/docker/image/image.go +++ b/pkg/minishift/docker/image/image.go @@ -27,10 +27,8 @@ import ( type ImageMissStrategy int const ( - SKIP ImageMissStrategy = iota - PULL - // TODO Implement a retry strategy as well (HF) - //RETRY + Skip ImageMissStrategy = iota + Pull ) type ImageCacheConfig struct { @@ -49,16 +47,20 @@ func GetOpenShiftImageNames(version string) []string { } } -func CreateExportCommand(version string) (*exec.Cmd, error) { +func CreateExportCommand(version string, profile string, images []string) (*exec.Cmd, error) { cmd, err := os.CurrentExecutable() if err != nil { return nil, err } exportArgs := []string{ + "--profile", + profile, "image", - "export"} - exportArgs = append(exportArgs, GetOpenShiftImageNames(version)...) + "export", + "--log-to-file", + } + exportArgs = append(exportArgs, images...) exportCmd := exec.Command(cmd, exportArgs...) // don't inherit any file handles exportCmd.Stderr = nil diff --git a/pkg/minishift/docker/image/image_handler.go b/pkg/minishift/docker/image/image_handler.go index af0c133121..0653cc8b89 100644 --- a/pkg/minishift/docker/image/image_handler.go +++ b/pkg/minishift/docker/image/image_handler.go @@ -16,341 +16,23 @@ limitations under the License. package image -import ( - "bufio" - "bytes" - "errors" - "fmt" - "github.com/docker/machine/libmachine/drivers" - "github.com/minishift/minishift/pkg/minikube/sshutil" - "github.com/minishift/minishift/pkg/util" - "github.com/minishift/minishift/pkg/util/filehelper" - minishiftStrings "github.com/minishift/minishift/pkg/util/strings" - "golang.org/x/crypto/ssh" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - "time" -) - -const ( - // Repository name seperator for container images - RepositorySeparator = '/' - - // Tag separator for container images - TagSeparator = ':' - - // Used to separate image name parts in a single file name - FileSeparator = "@" -) - // ImageHandler is responsible for the import and export of images into the Docker daemon of the VM type ImageHandler interface { - // Imports cached images from the host into the Docker daemon of the VM. + // ImportImages imports cached images from the host into the Docker daemon of the VM. ImportImages(config *ImageCacheConfig) error - // Exports the images specified as part of the ImageCacheConfig from the VM to the host. + // ExportImages exports the images specified as part of the ImageCacheConfig from the VM to the host. ExportImages(config *ImageCacheConfig) error + // IsImageCached returns true if the specified image is cached, false otherwise. + IsImageCached(config *ImageCacheConfig, image string) bool + // AreImagesCached returns true if all images specified in the config are cached, false otherwise. AreImagesCached(config *ImageCacheConfig) bool -} - -type DockerImageHandler struct { - driver drivers.Driver -} - -func NewDockerImageHandler(driver drivers.Driver) (*DockerImageHandler, error) { - return &DockerImageHandler{driver: driver}, nil -} - -func (handler *DockerImageHandler) ImportImages(config *ImageCacheConfig) error { - out := handler.getOutputWriter(config) - - files, _ := ioutil.ReadDir(config.HostCacheDir) - if len(files) > 0 { - fmt.Fprintln(out, "-- Importing cached images ...") - } - - for _, f := range files { - if strings.HasSuffix(f.Name(), ".tmp") { - continue - } - - if !minishiftStrings.Contains(config.CachedImages, handler.fileNameToImageName(f.Name())) { - continue - } - - err := handler.importImage(filepath.Join(config.HostCacheDir, f.Name()), out) - if err != nil { - return err - } - } - return nil -} - -func (handler *DockerImageHandler) ExportImages(config *ImageCacheConfig) error { - out := handler.getOutputWriter(config) - for _, exportImage := range config.CachedImages { - pulled, err := handler.IsPulled(exportImage) - if err != nil { - return err - } - - if !pulled { - if config.ImageMissStrategy == PULL { - err := handler.pullImage(exportImage, config.Out) - if err != nil { - return err - } - } else { - continue - } - } - - err = handler.exportImage(exportImage, config.HostCacheDir, out) - if err != nil { - return err - } - } - - return nil -} - -func (handler *DockerImageHandler) AreImagesCached(config *ImageCacheConfig) bool { - for _, image := range config.CachedImages { - imageCacheFile := filepath.Join(config.HostCacheDir, handler.imageNameToFileName(image)) - if !handler.isCached(imageCacheFile) { - return false - } - - } - - return true -} - -// IsPulled returns true is the specified image is already cached in the Docker daemon, false otherwise. -func (handler *DockerImageHandler) IsPulled(image string) (bool, error) { - session, err := handler.createSshSession() - if err != nil { - return false, err - } - defer session.Close() - - cmd := fmt.Sprintf("docker images -q %s", image) - var buffer bytes.Buffer - session.Stdout = &buffer - err = session.Run(cmd) - if err != nil { - return false, errors.New(fmt.Sprintf("Error running command '%s': %v", cmd, err)) - } - - if len(buffer.String()) > 0 { - return true, nil - } else { - return false, nil - } -} - -func (handler *DockerImageHandler) getOutputWriter(config *ImageCacheConfig) io.Writer { - var w io.Writer - if config.Out != nil { - w = config.Out - } else { - w = os.Stdout - } - return w -} - -func (handler *DockerImageHandler) importImage(imagePath string, out io.Writer) error { - defer util.TimeTrack(time.Now(), out, true) - fmt.Fprint(out, fmt.Sprintf(" Importing %s ", handler.fileNameToImageName(imagePath))) - - session, err := handler.createSshSession() - if err != nil { - return err - } - - defer session.Close() - - w, err := session.StdinPipe() - if err != nil { - return err - } - - errorCh := make(chan error) - go func() { - file, err := os.Open(imagePath) - if err != nil { - errorCh <- errors.New(fmt.Sprintf("Unable to open image: %v", err)) - } - - reader := bufio.NewReader(file) - io.Copy(w, reader) - w.Close() - - errorCh <- nil - }() - - cmd := "docker load" - err = session.Run(cmd) - if err != nil { - return errors.New(fmt.Sprintf("Error running command '%s': %v", cmd, err)) - } - - err = <-errorCh - if err != nil { - return err - } - - return nil -} - -func (handler *DockerImageHandler) exportImage(image string, cacheDir string, w io.Writer) error { - defer util.TimeTrack(time.Now(), w, true) - fmt.Fprint(w, fmt.Sprintf("Exporting image %s ", image)) - - imageCacheFile := filepath.Join(cacheDir, handler.imageNameToFileName(image)) - if handler.isCached(imageCacheFile) { - fmt.Fprint(w, " [already cached] ") - return nil - } - - session, err := handler.createSshSession() - if err != nil { - return err - } - defer session.Close() - - r, err := session.StdoutPipe() - if err != nil { - return err - } - - errorCh := make(chan error) - go func() { - file, err := os.Create(imageCacheFile + ".tmp") - if err != nil { - errorCh <- errors.New(fmt.Sprintf("Unable to create temporary image file: %v", err)) - } - - w := bufio.NewWriter(file) - io.Copy(w, r) - - w.Flush() - file.Close() - - err = os.Rename(file.Name(), imageCacheFile) - if err != nil { - errorCh <- errors.New(fmt.Sprintf("Unable to create image file: %v", err)) - } - - errorCh <- nil - }() - - cmd := fmt.Sprintf("docker save %s", image) - err = session.Run(cmd) - if err != nil { - return errors.New(fmt.Sprintf("Error running command '%s': %v", cmd, err)) - } - - err = <-errorCh - if err != nil { - return err - } - - return nil -} - -func (handler *DockerImageHandler) pullImage(image string, w io.Writer) error { - defer util.TimeTrack(time.Now(), w, true) - fmt.Fprint(w, fmt.Sprintf("Pulling image %s ", image)) - - session, err := handler.createSshSession() - if err != nil { - return err - } - defer session.Close() - - cmd := fmt.Sprintf("docker pull %s", image) - err = session.Run(cmd) - if err != nil { - return errors.New(fmt.Sprintf("Error running command '%s': %v", cmd, err)) - } - - return nil -} - -// availableImages returns a slice of available images in the current Docker instance -func (handler *DockerImageHandler) availableImages() ([]string, error) { - session, err := handler.createSshSession() - if err != nil { - return nil, err - } - defer session.Close() - - cmd := "docker images --format '{{.Repository}}:{{.Tag}}'" - var buffer bytes.Buffer - session.Stdout = &buffer - err = session.Run(cmd) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error running command '%s': %v", cmd, err)) - } - - availableImages := strings.Split(buffer.String(), "\n") - return availableImages, nil -} - -// imageNameToFileName maps an image name in the format repository/name:tag to a file system name. -func (handler *DockerImageHandler) imageNameToFileName(image string) string { - imageNameParts := strings.FieldsFunc(image, func(r rune) bool { - switch r { - case RepositorySeparator, TagSeparator: - return true - } - return false - }) - - return strings.Join(imageNameParts, FileSeparator) -} - -// isImageFile returns true if the denoted path represents a image file (based on file name). -func (handler *DockerImageHandler) isImageFile(path string) bool { - fileName := filepath.Base(path) - match, err := regexp.Match("^.*@.*@.*$", []byte(fileName)) - if err != nil { - return false - } - return match -} - -// fileNameToImageName maps a filename on the host to an image name in the format repository/name:tag. -func (handler *DockerImageHandler) fileNameToImageName(path string) string { - fileName := filepath.Base(path) - parts := strings.Split(fileName, FileSeparator) - - return parts[0] + string(RepositorySeparator) + parts[1] + string(TagSeparator) + parts[2] -} - -// isCached returns true if the specified image (file) exists on the host. -func (handler *DockerImageHandler) isCached(imageCacheFile string) bool { - return filehelper.Exists(imageCacheFile) -} - -// createSshSession creates an interactive SSH session -func (handler *DockerImageHandler) createSshSession() (*ssh.Session, error) { - sshClient, err := sshutil.NewSSHClient(handler.driver) - if err != nil { - return nil, err - } - session, err := sshClient.NewSession() - if err != nil { - return nil, err - } + // GetCachedImages returns a map of cached image names. A map is used to make lookup for a specific image easier. + GetCachedImages(config *ImageCacheConfig) map[string]bool - return session, nil + // GetDockerImages returns a map of images available in the Docker daemon. A map is used to make lookup for a specific image easier. + GetDockerImages() (map[string]bool, error) } diff --git a/pkg/minishift/docker/image/image_handler_test.go b/pkg/minishift/docker/image/image_handler_test.go deleted file mode 100644 index 941455b146..0000000000 --- a/pkg/minishift/docker/image/image_handler_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright (C) 2017 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package image - -import ( - "github.com/minishift/minishift/pkg/minikube/tests" - "testing" -) - -var testHandler *DockerImageHandler - -type imageTestCase struct { - fileName string - isImage bool -} - -func Test_converting_image_name_to_file_name(t *testing.T) { - setUp(t) - - actualFileName := testHandler.imageNameToFileName("openshift/origin:v1.5.1") - expectedFileName := "openshift@origin@v1.5.1" - if actualFileName != expectedFileName { - t.Fatalf("Expected '%s', but got '%s'", expectedFileName, actualFileName) - } -} - -func Test_converting_file_name_to_image_name(t *testing.T) { - setUp(t) - - actualImageName := testHandler.fileNameToImageName("/foo/bar/openshift@origin@v1.5.1") - expectedImageName := "openshift/origin:v1.5.1" - if actualImageName != expectedImageName { - t.Fatalf("Expected '%s', but got '%s'", expectedImageName, actualImageName) - } -} - -func Test_is_image_name(t *testing.T) { - setUp(t) - - testData := []imageTestCase{ - {fileName: "openshift@origin@v1.5.1", isImage: true}, - {fileName: "openshift@origin:v1.5.1", isImage: false}, - {fileName: "openshift/origin:v1.5.1", isImage: false}, - } - - for _, testCase := range testData { - isImage := testHandler.isImageFile(testCase.fileName) - if testCase.isImage != isImage { - t.Fatalf("Expected isImageFile for '%s' to return %t, but got %t", testCase.fileName, testCase.isImage, isImage) - } - - } -} - -func setUp(t *testing.T) { - var err error - testHandler, err = NewDockerImageHandler(&tests.MockDriver{}) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } -} diff --git a/pkg/minishift/docker/image/oci_image_handler.go b/pkg/minishift/docker/image/oci_image_handler.go new file mode 100644 index 0000000000..a4d7b0c357 --- /dev/null +++ b/pkg/minishift/docker/image/oci_image_handler.go @@ -0,0 +1,397 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "errors" + "fmt" + "io" + "os" + "strconv" + + "bytes" + "encoding/json" + "github.com/containers/image/copy" + "github.com/containers/image/oci/layout" + "github.com/containers/image/signature" + "github.com/containers/image/transports/alltransports" + "github.com/containers/image/types" + "github.com/docker/machine/libmachine/drivers" + "github.com/minishift/minishift/pkg/minikube/sshutil" + "github.com/minishift/minishift/pkg/util/filehelper" + "github.com/minishift/minishift/pkg/util/progressdots" + "golang.org/x/crypto/ssh" + "io/ioutil" + "path/filepath" + "strings" +) + +// OciImageHandler is an ImageHandler implementation using OCI format to maintain the local cache. +type OciImageHandler struct { + driver drivers.Driver + dockerClientSettings *dockerClientConfig +} + +type dockerClientConfig struct { + DockerHost string + DockerCertPath string + DockerTLSVerify bool +} + +type Index struct { + Manifests Manifests `json:"manifests"` +} + +type Manifests []Manifest + +type Manifest struct { + Annotations Annotations `json:"annotations"` +} + +type Annotations struct { + Name string `json:"org.opencontainers.image.ref.name"` +} + +// NewOciImageHandler creates a new ImageHandler which stores cached images in OCI format. +// It takes a reference to a Driver in order to communicate with the VM and Docker and a map containing the environment settings for the Minishift Docker daemon. +func NewOciImageHandler(driver drivers.Driver, dockerEnv map[string]string) (*OciImageHandler, error) { + settings, err := getDockerSettings(dockerEnv) + if err != nil { + return nil, err + } + return &OciImageHandler{driver: driver, dockerClientSettings: settings}, nil +} + +// NewLocalOnlyOciImageHandler creates a new ImageHandler which can only interact with the local cache. +// No connection information to the Docker daemon are provided. Functions interacting with the Docker daemon will fail. +func NewLocalOnlyOciImageHandler() (*OciImageHandler, error) { + return &OciImageHandler{driver: nil, dockerClientSettings: nil}, nil +} + +// ImportImages imports cached images from the host into the Docker daemon of the VM. +func (handler *OciImageHandler) ImportImages(config *ImageCacheConfig) error { + out := handler.getOutputWriter(config) + + policyContext, err := handler.getPolicyContext() + if err != nil { + return fmt.Errorf("Error creating security context: %s", err.Error()) + } + + availableImages, err := handler.GetDockerImages() + if err != nil { + return err + } + + for _, imageName := range config.CachedImages { + fmt.Fprint(out, fmt.Sprintf(" Importing %s ", imageName)) + progressDots := progressdots.New() + progressDots.SetWriter(out) + progressDots.Start() + if _, found := availableImages[imageName]; found { + handler.endProgress(progressDots, out, nil) + continue + + } + + if !handler.IsImageCached(config, imageName) { + handler.endProgress(progressDots, out, nil) + continue + } + + err := handler.importImage(imageName, config, policyContext, out) + handler.endProgress(progressDots, out, err) + if err != nil { + return err + } + + } + return nil +} + +// ExportImages exports the images specified as part of the ImageCacheConfig from the VM to the host. +func (handler *OciImageHandler) ExportImages(config *ImageCacheConfig) error { + out := handler.getOutputWriter(config) + + policyContext, err := handler.getPolicyContext() + if err != nil { + return fmt.Errorf("Error creating security context: %s", err.Error()) + } + + for _, imageName := range config.CachedImages { + fmt.Fprint(out, fmt.Sprintf("Exporting %s", imageName)) + progressDots := progressdots.New() + progressDots.SetWriter(out) + progressDots.Start() + + if !handler.IsImageCached(config, imageName) { + err = handler.exportImage(imageName, config, policyContext, out) + } + handler.endProgress(progressDots, out, err) + if err != nil { + return err + } + } + + return nil +} + +// IsImageCached returns true if the specified image is cached, false otherwise. +func (handler *OciImageHandler) IsImageCached(config *ImageCacheConfig, image string) bool { + cachedImages := handler.GetCachedImages(config) + _, found := cachedImages[image] + return found +} + +// AreImagesCached returns true if all images specified in the config are cached, false otherwise. +func (handler *OciImageHandler) AreImagesCached(config *ImageCacheConfig) bool { + cachedImages := handler.GetCachedImages(config) + + for _, image := range config.CachedImages { + if _, found := cachedImages[image]; !found { + return false + } + } + + return true +} + +func (handler *OciImageHandler) GetCachedImages(config *ImageCacheConfig) map[string]bool { + cachedImages := make(map[string]bool) + + index, err := handler.getIndex(config) + if index == nil || err != nil { + return cachedImages + } + + for _, manifest := range index.Manifests { + cachedImages[manifest.Annotations.Name] = true + } + + return cachedImages +} + +func (handler *OciImageHandler) GetDockerImages() (map[string]bool, error) { + dockerImages := make(map[string]bool) + + session, err := handler.createSSHSession() + if err != nil { + return nil, err + } + defer session.Close() + + cmd := "docker images --format '{{.Repository}}:{{.Tag}}'" + var buffer bytes.Buffer + session.Stdout = &buffer + err = session.Run(cmd) + if err != nil { + return nil, fmt.Errorf("Error running command '%s': %v", cmd, err) + } + + for _, image := range strings.Split(buffer.String(), "\n") { + if len(image) > 0 { + dockerImages[image] = true + } + } + + return dockerImages, nil +} + +func (handler *OciImageHandler) pullImage(image string, out io.Writer) error { + fmt.Fprint(out, fmt.Sprintf("Pulling image %s ", image)) + + session, err := handler.createSSHSession() + if err != nil { + return err + } + defer session.Close() + + progressDots := progressdots.New() + progressDots.SetWriter(out) + progressDots.Start() + defer handler.endProgress(progressDots, out, err) + cmd := fmt.Sprintf("docker pull %s", image) + cmdOut, err := session.CombinedOutput(cmd) + if err != nil { + return fmt.Errorf("Error running command '%s': %v \n%s", cmd, err, string(cmdOut[:])) + } + + return nil +} + +func (handler *OciImageHandler) getIndex(config *ImageCacheConfig) (*Index, error) { + indexPath := filepath.Join(config.HostCacheDir, "index.json") + if !filehelper.Exists(indexPath) { + return nil, nil + } + + raw, err := ioutil.ReadFile(indexPath) + if err != nil { + return nil, err + } + + var index Index + err = json.Unmarshal(raw, &index) + if err != nil { + return nil, err + } + + return &index, nil +} + +func (handler *OciImageHandler) importImage(image string, config *ImageCacheConfig, policyContext *signature.PolicyContext, out io.Writer) error { + srcRef, err := layout.NewReference(config.HostCacheDir, image) + if err != nil { + return fmt.Errorf("Invalid image source '%v': %v", srcRef, err) + } + + destRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-daemon:%s", image)) + if err != nil { + return fmt.Errorf("Invalid image source '%s': %v", image, err) + } + + err = handler.copyImage(srcRef, destRef, policyContext) + if err != nil { + return err + } + + return nil +} + +func (handler *OciImageHandler) exportImage(image string, config *ImageCacheConfig, policyContext *signature.PolicyContext, out io.Writer) error { + availableImages, err := handler.GetDockerImages() + if err != nil { + return err + } + + if _, found := availableImages[image]; !found { + err := handler.pullImage(image, config.Out) + if err != nil { + return err + } + } + + srcRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-daemon:%s", image)) + if err != nil { + return fmt.Errorf("Invalid image source '%s': %v", image, err) + } + + destRef, err := layout.NewReference(config.HostCacheDir, image) + if err != nil { + return fmt.Errorf("Invalid image destination '%v': %v", destRef, err) + } + + err = handler.copyImage(srcRef, destRef, policyContext) + if err != nil { + return err + } + + return nil +} + +func (handler *OciImageHandler) copyImage(srcRef types.ImageReference, destRef types.ImageReference, policyContext *signature.PolicyContext) error { + err := copy.Image(policyContext, destRef, srcRef, ©.Options{ + RemoveSignatures: false, + SignBy: "", + ReportWriter: nil, + SourceCtx: handler.getSystemContext(), + DestinationCtx: handler.getSystemContext(), + }) + if err != nil { + return err + } + + return nil +} + +func (handler *OciImageHandler) getOutputWriter(config *ImageCacheConfig) io.Writer { + var w io.Writer + if config.Out != nil { + w = config.Out + } else { + w = os.Stdout + } + return w +} + +func (handler *OciImageHandler) getSystemContext() *types.SystemContext { + return &types.SystemContext{ + DockerDaemonHost: handler.dockerClientSettings.DockerHost, + DockerDaemonCertPath: handler.dockerClientSettings.DockerCertPath, + DockerDaemonInsecureSkipTLSVerify: !handler.dockerClientSettings.DockerTLSVerify, + } +} + +// createSSHSession creates an interactive SSH session +func (handler *OciImageHandler) createSSHSession() (*ssh.Session, error) { + sshClient, err := sshutil.NewSSHClient(handler.driver) + if err != nil { + return nil, err + } + + session, err := sshClient.NewSession() + if err != nil { + return nil, err + } + + return session, nil +} + +func (handler *OciImageHandler) getPolicyContext() (*signature.PolicyContext, error) { + policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error creating security context: %s", err.Error())) + } + + return policyContext, nil +} + +func (handler *OciImageHandler) endProgress(progressDots *progressdots.ProgressDots, out io.Writer, err error) { + progressDots.Stop() + if err == nil { + fmt.Fprint(out, " OK") + } + fmt.Fprint(out, "\n") +} + +func getDockerSettings(dockerEnv map[string]string) (*dockerClientConfig, error) { + settings := &dockerClientConfig{} + + if val, ok := dockerEnv["DOCKER_HOST"]; ok { + settings.DockerHost = val + } else { + return nil, errors.New("The provided Docker environment settings are missing the DOCKER_HOST key.") + } + + if val, ok := dockerEnv["DOCKER_CERT_PATH"]; ok { + settings.DockerCertPath = val + } else { + return nil, errors.New("The provided Docker environment settings are missing the DOCKER_CERT_PATH key.") + } + + if val, ok := dockerEnv["DOCKER_TLS_VERIFY"]; ok { + verify, err := strconv.ParseBool(val) + if err != nil { + return nil, errors.New(fmt.Sprintf("Invalid value '%s' for DOCKER_TLS_VERIFY key.", val)) + } + settings.DockerTLSVerify = verify + } else { + return nil, errors.New("The provided Docker environment settings are missing the DOCKER_TLS_VERIFY key.") + } + + return settings, nil +} diff --git a/pkg/minishift/docker/image/oci_image_handler_test.go b/pkg/minishift/docker/image/oci_image_handler_test.go new file mode 100644 index 0000000000..bfd000b1ae --- /dev/null +++ b/pkg/minishift/docker/image/oci_image_handler_test.go @@ -0,0 +1,88 @@ +/* +Copyright (C) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func Test_Are_Images_Cached(t *testing.T) { + currDir, err := os.Getwd() + if err != nil { + t.Fatal("Unable to determine working directory.") + } + + cacheConfig := &ImageCacheConfig{ + HostCacheDir: filepath.Join(currDir, "testdata"), + CachedImages: []string{"openshift/origin:v3.6.0", "openshift/origin-pod:v3.6.0", "openshift/origin-docker-registry:v3.6.0", "openshift/origin-haproxy-router:v3.6.0"}, + Out: nil, + ImageMissStrategy: Skip, + } + + handler := OciImageHandler{} + allCached := handler.AreImagesCached(cacheConfig) + if !allCached { + t.Fatal("According to the index all images should be cached") + } + + cacheConfig = &ImageCacheConfig{ + HostCacheDir: filepath.Join(currDir, "testdata"), + CachedImages: []string{"foo/bar:v3.6.0"}, + Out: nil, + ImageMissStrategy: Skip, + } + + handler = OciImageHandler{} + allCached = handler.AreImagesCached(cacheConfig) + if allCached { + t.Fatal("According to the index the image should not be cached") + } +} + +func Test_Get_Docker_Settings(t *testing.T) { + var envTests = []struct { + envMap map[string]string + dockerSettings *dockerClientConfig + errorMessage string + }{ + {nil, nil, "The provided Docker environment settings are missing the DOCKER_HOST key."}, + {map[string]string{}, nil, "The provided Docker environment settings are missing the DOCKER_HOST key."}, + {map[string]string{"DOCKER_HOST": "foo"}, nil, "The provided Docker environment settings are missing the DOCKER_CERT_PATH key."}, + {map[string]string{"DOCKER_HOST": "foo", "DOCKER_CERT_PATH": "foo"}, nil, "The provided Docker environment settings are missing the DOCKER_TLS_VERIFY key."}, + {map[string]string{"DOCKER_HOST": "foo", "DOCKER_CERT_PATH": "bar", "DOCKER_TLS_VERIFY": "1"}, &dockerClientConfig{DockerHost: "foo", DockerCertPath: "bar", DockerTLSVerify: true}, ""}, + } + + for _, envTest := range envTests { + clientConfig, err := getDockerSettings(envTest.envMap) + if err != nil { + if envTest.errorMessage != "" { + if err.Error() != envTest.errorMessage { + t.Errorf("Unexpected error message. Expected '%s'. Got '%s'", envTest.errorMessage, err.Error()) + } + } else { + t.Errorf("There was no error expected. Got '%v'", err) + } + } + + if !reflect.DeepEqual(clientConfig, envTest.dockerSettings) { + t.Errorf("Expected and received settings don't match. Got '%v'. Expected '%v'", clientConfig, envTest.dockerSettings) + } + } +} diff --git a/pkg/minishift/docker/image/testdata/index.json b/pkg/minishift/docker/image/testdata/index.json new file mode 100644 index 0000000000..b518cd99ab --- /dev/null +++ b/pkg/minishift/docker/image/testdata/index.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:bb1fb6e9d0509afa26e52b75bdf144faea07d2473c9d5c693427b6f09b6c3ba5", + "size": 823, + "annotations": { + "org.opencontainers.image.ref.name": "openshift/origin:v3.6.0" + }, + "platform": { + "architecture": "amd64", + "os": "darwin" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:3f35865b0dd264757de97eb62f1f317791e9fcd63e647dd47b5f73cd012e9aef", + "size": 662, + "annotations": { + "org.opencontainers.image.ref.name": "openshift/origin-pod:v3.6.0" + }, + "platform": { + "architecture": "amd64", + "os": "darwin" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:a6f02728622186556d5842e44cd73b5e8b81ac43025e629e7a120351f23b41ea", + "size": 823, + "annotations": { + "org.opencontainers.image.ref.name": "openshift/origin-docker-registry:v3.6.0" + }, + "platform": { + "architecture": "amd64", + "os": "darwin" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:5a1f719285eb1aaf9459acfe5c4b01d7813426e36f09b61d965f1a86141a74ef", + "size": 981, + "annotations": { + "org.opencontainers.image.ref.name": "openshift/origin-haproxy-router:v3.6.0" + }, + "platform": { + "architecture": "amd64", + "os": "darwin" + } + } + ] +} diff --git a/pkg/util/os/process/sysproc_darwin.go b/pkg/util/os/process/sysproc_darwin.go index 30c99c7161..738c1ca8e8 100644 --- a/pkg/util/os/process/sysproc_darwin.go +++ b/pkg/util/os/process/sysproc_darwin.go @@ -33,7 +33,7 @@ func SysProcForBackgroundProcess() *syscall.SysProcAttr { func EnvForBackgroundProcess() []string { return []string{ - fmt.Sprintf("MINISHIFT_HOME=%s", constants.Minipath), + fmt.Sprintf("MINISHIFT_HOME=%s", constants.GetMinishiftHomeDir()), fmt.Sprintf("PATH=%s", os.Getenv("PATH")), } } diff --git a/pkg/util/os/process/sysproc_linux.go b/pkg/util/os/process/sysproc_linux.go index 30c99c7161..738c1ca8e8 100644 --- a/pkg/util/os/process/sysproc_linux.go +++ b/pkg/util/os/process/sysproc_linux.go @@ -33,7 +33,7 @@ func SysProcForBackgroundProcess() *syscall.SysProcAttr { func EnvForBackgroundProcess() []string { return []string{ - fmt.Sprintf("MINISHIFT_HOME=%s", constants.Minipath), + fmt.Sprintf("MINISHIFT_HOME=%s", constants.GetMinishiftHomeDir()), fmt.Sprintf("PATH=%s", os.Getenv("PATH")), } } diff --git a/pkg/util/os/process/sysproc_windows.go b/pkg/util/os/process/sysproc_windows.go index 8df10b3b1a..f9b8202af7 100644 --- a/pkg/util/os/process/sysproc_windows.go +++ b/pkg/util/os/process/sysproc_windows.go @@ -36,10 +36,12 @@ func SysProcForBackgroundProcess() *syscall.SysProcAttr { func EnvForBackgroundProcess() []string { return []string{ - fmt.Sprintf("MINISHIFT_HOME=%s", constants.Minipath), + fmt.Sprintf("MINISHIFT_HOME=%s", constants.GetMinishiftHomeDir()), fmt.Sprintf("PATH=%s", os.Getenv("PATH")), fmt.Sprintf("PATHEXT=%s", os.Getenv("PATHEXT")), fmt.Sprintf("SystemRoot=%s", os.Getenv("SystemRoot")), fmt.Sprintf("COMPUTERNAME=%s", os.Getenv("COMPUTERNAME")), + fmt.Sprintf("TMP=%s", os.Getenv("TMP")), + fmt.Sprintf("TEMP=%s", os.Getenv("TEMP")), } } diff --git a/pkg/util/progressdots/progressdots.go b/pkg/util/progressdots/progressdots.go index 44fb9d28bf..6928e7b183 100644 --- a/pkg/util/progressdots/progressdots.go +++ b/pkg/util/progressdots/progressdots.go @@ -18,6 +18,8 @@ package progressdots import ( "fmt" + "io" + "os" "time" ) @@ -31,6 +33,7 @@ type ProgressDots struct { easing int dotCounter int handler chan bool + out io.Writer } // New creates the channel to handle progress dots @@ -45,6 +48,7 @@ func New(easingOptional ...int) *ProgressDots { easing: easing, dotCounter: 0, handler: make(chan bool), + out: os.Stdout, } } @@ -56,7 +60,7 @@ func (s *ProgressDots) Start() { case <-s.handler: return default: - fmt.Print(".") + fmt.Fprint(s.out, ".") s.dotCounter++ time.Sleep(s.interval) if s.easing != 0 && s.dotCounter%s.easing == 0 { @@ -78,3 +82,8 @@ func (s *ProgressDots) Stop() { func (s *ProgressDots) SetInterval(interval time.Duration) { s.interval = interval } + +// SetWriter sets the writer for the output +func (s *ProgressDots) SetWriter(out io.Writer) { + s.out = out +} diff --git a/pkg/util/strings/strings.go b/pkg/util/strings/strings.go index 16f045bd92..f78add38eb 100644 --- a/pkg/util/strings/strings.go +++ b/pkg/util/strings/strings.go @@ -38,6 +38,17 @@ func Contains(slice []string, s string) bool { return false } +// Remove takes a slice os strings and returns a slice with the first occurance of the string value removed. +func Remove(slice []string, value string) []string { + for i, s := range slice { + if s == value { + slice = append(slice[:i], slice[i+1:]...) + break + } + } + return slice +} + func EscapeSingleQuote(s string) string { r := strings.NewReplacer(`'`, `'"'"'`) return r.Replace(s) diff --git a/pkg/util/strings/strings_test.go b/pkg/util/strings/strings_test.go index 99682f3ccb..cb1480b04d 100644 --- a/pkg/util/strings/strings_test.go +++ b/pkg/util/strings/strings_test.go @@ -46,6 +46,24 @@ func TestContains(t *testing.T) { } } +func TestRemove(t *testing.T) { + var testCases = []struct { + slice []string + element string + expected []string + }{ + {[]string{"a", "b", "c"}, "b", []string{"a", "c"}}, + {[]string{"a", "b", "c"}, "", []string{"a", "b", "c"}}, + {[]string{"a", "b", "c"}, "d", []string{"a", "b", "c"}}, + {[]string{}, "a", []string{}}, + } + + for _, testCase := range testCases { + actual := Remove(testCase.slice, testCase.element) + minishiftTesting.AssertEqualSlice(testCase.expected, actual, t) + } +} + func TestHasMatcher(t *testing.T) { var testCases = []struct { testString string diff --git a/test/integration/features/basic.feature b/test/integration/features/basic.feature index 46d92b6ebe..0e14a03912 100644 --- a/test/integration/features/basic.feature +++ b/test/integration/features/basic.feature @@ -3,7 +3,6 @@ Feature: Basic As a user I can perform basic operations of Minishift and OpenShift Scenario: User can install default add-ons - Given Minishift has state "Does Not Exist" When executing "minishift addons install --defaults" succeeds Then stdout should contain """ @@ -11,7 +10,6 @@ Feature: Basic """ Scenario: User can enable the anyuid add-on - Given Minishift has state "Does Not Exist" When executing "minishift addons enable anyuid" succeeds Then stdout should contain """ @@ -20,7 +18,6 @@ Feature: Basic @minishift-only Scenario: User can list enabled add-ons - Given Minishift has state "Does Not Exist" When executing "minishift addons list" succeeds Then stdout should contain """ @@ -36,9 +33,10 @@ Feature: Basic Scenario: OpenShift is ready after startup After startup of Minishift OpenShift instance should respond correctly on its html endpoints and OpenShift web console should be accessible. + Given Minishift has state "Running" - Then status code of HTTP request to "OpenShift" at "/healthz" is equal to "200" - And body of HTTP request to "OpenShift" at "/healthz" contains "ok" + When status code of HTTP request to "OpenShift" at "/healthz" is equal to "200" + Then body of HTTP request to "OpenShift" at "/healthz" contains "ok" And status code of HTTP request to "OpenShift" at "/healthz/ready" is equal to "200" And body of HTTP request to "OpenShift" at "/healthz/ready" contains "ok" And status code of HTTP request to "OpenShift" at "/console" is equal to "200" @@ -47,11 +45,13 @@ Feature: Basic Scenario Outline: User can set, get, view and unset values in configuration file User is able to setup persistent configuration of Minishift using "config" command and its subcommands, changing values stored in "config/config.json". + Given Minishift has state "Running" When executing "minishift config set " succeeds Then JSON config file "config/config.json" contains key "" with value matching "" And stdout of command "minishift config get " is equal to "" And stdout of command "minishift config view --format {{.ConfigKey}}:{{.ConfigValue}}" contains ":" + When executing "minishift config unset " succeeds Then stdout of command "minishift config get " is equal to "" And JSON config file "config/config.json" does not have key "" @@ -64,6 +64,7 @@ Feature: Basic Scenario: User can get IP of provided virtual machine User is able to get IP of Minishift VM with command "minishift ip". + Given Minishift has state "Running" When executing "minishift ip" succeeds Then stdout should be valid IP @@ -78,6 +79,7 @@ Feature: Basic Scenario: OpenShift developer has sudo permissions The 'developer' user should be configured with the sudoer role after starting Minishift + Given Minishift has state "Running" When executing "oc --as system:admin get clusterrolebindings" succeeds Then stdout should contain @@ -91,6 +93,7 @@ Feature: Basic Scenario: A 'minishift' context is created for 'oc' usage After a successful Minishift start the user's current context is 'minishift' + Given Minishift has state "Running" When executing "oc config current-context" succeeds Then stdout should contain @@ -135,66 +138,36 @@ Feature: Basic 172.30.1.1:5000 """ - # User can deploy the example Ruby application ruby-ex - Scenario: User can login to the server + Scenario: User can deploy a Ruby example application Given Minishift has state "Running" When executing "oc login --username=developer --password=developer" succeeds - Then stdout should contain - """ - Login successful - """ - - Scenario: User can create new namespace ruby for application ruby-ex - Given Minishift has state "Running" - When executing "oc new-project ruby" succeeds - Then stdout should contain - """ - Now using project "ruby" - """ - - Scenario: User can deploy application ruby-ex to namespace ruby - Given Minishift has state "Running" - When executing "oc new-app centos/ruby-22-centos7~https://github.com/openshift/ruby-ex.git" succeeds - Then stdout should contain - """ - Success - """ + And executing "oc new-project ruby" succeeds + And executing "oc new-app centos/ruby-22-centos7~https://github.com/openshift/ruby-ex.git" succeeds And services "ruby-ex" rollout successfully + And executing "oc expose svc/ruby-ex" succeeds + Then status code of HTTP request to "/" of service "ruby-ex" in namespace "ruby" is equal to "200" + And body of HTTP request to "/" of service "ruby-ex" in namespace "ruby" contains "Welcome to your Ruby application on OpenShift" - Scenario: User can create route for ruby-ex to make it visiable outside of the cluster - Given Minishift has state "Running" - When executing "oc expose svc/ruby-ex" succeeds - Then stdout should contain - """ - exposed - """ - And status code of HTTP request to "/" of service "ruby-ex" in namespace "ruby" is equal to "200" - And body of HTTP request to "/" of service "ruby-ex" in namespace "ruby" contains "Welcome to your Ruby application on OpenShift" - - Scenario: User can delete namespace ruby - Given Minishift has state "Running" When executing "oc delete project ruby" succeeds - Then stdout should contain - """ - "ruby" deleted - """ + Then stdout should contain "\"ruby\" deleted" - Scenario: User can log out the session - Given Minishift has state "Running" When executing "oc logout" succeeds Then stdout should contain """ Logged "developer" out """ - # End of Ruby application ruby-ex deployment + + Scenario: As a user I am able to export a container image from a running Minshift instance + Note: Just a sanity check for image caching. For more extensive tests see cmd-image.feature + + When executing "minishift image export alpine:latest" succeeds + Then stdout of command "minishift image list" contains "alpine:latest" Scenario: Stopping Minishift Given Minishift has state "Running" When executing "minishift stop" succeeds Then Minishift should have state "Stopped" - Scenario: Stopping an already stopped VM - Given Minishift has state "Stopped" When executing "minishift stop" Then Minishift should have state "Stopped" And stdout should contain @@ -204,7 +177,7 @@ Feature: Basic Scenario: Deleting Minishift Given Minishift has state "Stopped" - When executing "minishift delete --force" succeeds + When executing "minishift delete --force --clear-cache" succeeds Then Minishift should have state "Does Not Exist" When executing "minishift ip" Then exitcode should equal "1" diff --git a/test/integration/features/cmd-image.feature b/test/integration/features/cmd-image.feature new file mode 100644 index 0000000000..74a82cab33 --- /dev/null +++ b/test/integration/features/cmd-image.feature @@ -0,0 +1,80 @@ +@cmd-image @command +Feature: Basic image caching test + As a user I am able to import and export container images from a local OCI repository + located in the $MINISHIFT_HOME/cache directory + + Scenario: As a user I can export a container image from a running Minishift instance + Given Minishift has state "Does Not Exist" + When executing "minishift image list" succeeds + Then stdout should be empty + + When executing "minishift image export alpine:latest" succeeds + Then stdout should contain + """ + Running this command requires an existing 'minishift' VM, but no VM is defined. + """ + + When executing "minishift start" succeeds + And executing "minishift image export alpine:latest" succeeds + Then stdout of command "minishift image list" contains "alpine:latest" + + # Cache is retained + When executing "minishift delete --force" succeeds + Then stdout of command "minishift image list" contains "alpine:latest" + + Scenario: As a user I can import a container image from the local cache into a running Minishift instance + Note: In this scenario we use alpine:latest which was cached in the previous scenario + + Given Minishift has state "Does Not Exist" + When executing "minishift image list" succeeds + Then stdout should contain "alpine:latest" + + When executing "minishift image import alpine:latest" + Then stdout should contain + """ + Running this command requires an existing 'minishift' VM, but no VM is defined. + """ + + When executing "minishift start" succeeds + And executing "minishift image list --vm" succeeds + And stdout should not contain "alpine:latest" + And executing "minishift image import alpine:latest" succeeds + And executing "minishift image list --vm" succeeds + Then stdout should contain "alpine:latest" + + When executing "minishift delete --force" succeeds + Then Minishift should have state "Does Not Exist" + + Scenario: As a user I can enable implicit image + Implicit image caching means that a list of configured images is imported automatically/implicitly during 'minishift start'. + The user enables implicit image caching by setting the configuration property 'image-caching'. + The user also configures the images to be imported implicitly using the 'image config add' command. + + Given Minishift has state "Does Not Exist" + And executing "minishift config set image-caching true" succeeds + And executing "minishift image cache-config add alpine:latest" succeeds + Then JSON config file "config/config.json" contains key "image-caching" with value matching "true" + And stdout of command "minishift config get image-caching" is equal to "true" + And JSON config file "config/config.json" contains key "cache-images" with value matching "[alpine:latest]" + + When executing "minishift start" succeeds + Then stdout of command "minishift image list --vm" contains "alpine:latest" + + When executing "minishift delete --force --clear-cache" succeeds + Then Minishift should have state "Does Not Exist" + + Scenario: As a user I can view, remove and add the image cache configuration + Note: alpine:latest is already added to the list in a previous scenario + + Given stdout of command "minishift image cache-config view" contains "alpine:latest" + When executing "minishift image cache-config add busybox:latest" succeeds + Then stdout of command "minishift image cache-config view" contains "alpine:latest" + And stdout of command "minishift image cache-config view" contains "busybox:latest" + + When executing "minishift image cache-config remove alpine:latest" succeeds + Then stdout of command "minishift image cache-config view" does not contain "alpine:latest" + And stdout of command "minishift image cache-config view" contains "busybox:latest" + + When executing "minishift image cache-config remove busybox:latest" succeeds + And executing "minishift image cache-config view" succeeds + Then stdout should be empty