Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Issue minishift#952 Implementing image caching using OCI based image …
…cache

- Switching to containers/image OCI tansport for image caching
- Addressing cross compilation issues by making sure the right build tags are used
- Replacing current ImageHandler with an OCI based ImageHandler
- Making sure image caching work in conjunction with profiles (issue minishift#1666)
- Implementing image commands import, export and list
- Adding commands image config [view|add|remove] for configuring the list of persistent cache images
  • Loading branch information
hferentschik committed Dec 1, 2017
1 parent e993503 commit 52a24d7
Show file tree
Hide file tree
Showing 33 changed files with 1,945 additions and 580 deletions.
245 changes: 208 additions & 37 deletions Gopkg.lock

Large diffs are not rendered by default.

82 changes: 75 additions & 7 deletions Gopkg.toml
Expand Up @@ -20,6 +20,8 @@
# name = "github.com/x/y"
# version = "2.4.0"

# We don't want to end up with an old logrus import, only github.com/sirupsen/logrus should be used
ignored = ["github.com/Sirupsen/logrus"]

[[constraint]]
name = "github.com/DATA-DOG/godog"
Expand All @@ -35,13 +37,15 @@

[[constraint]]
name = "github.com/docker/go-units"
revision = "0bbddae09c5a5419a8c6dcdd7ff90da3d450393b"

[[constraint]]
name = "github.com/docker/machine"
version = "=0.11.0"

[[constraint]]
name = "github.com/elazarl/goproxy"
revision = "aacba83f36a55ac31cbb71c06547a328c0cd1604"

[[constraint]]
name = "github.com/gbraad/go-hvkvp"
Expand All @@ -62,42 +66,106 @@

[[constraint]]
name = "github.com/inconshreveable/go-update"
revision = "8152e7eb6ccf8679a64582a66b78519688d156ad"

[[constraint]]
name = "github.com/kardianos/osext"

[[constraint]]
name = "github.com/olekukonko/tablewriter"

[[constraint]]
name = "github.com/pborman/uuid"
revision = "febf2d34b54a69ce7530036c7503b1c9fbfdf0bb"

[[constraint]]
name = "github.com/pkg/browser"
revision = "c90ca0c84f15f81c982e32665bffd8d7aac8f097"

[[constraint]]
name = "github.com/pkg/errors"
version = "0.8.0"
version = "=0.8.0"

[[constraint]]
name = "github.com/spf13/cobra"
revision = "e606913c4ee45fec232e67e70105fb6c866b95d9"

[[constraint]]
name = "github.com/spf13/pflag"
revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"

[[constraint]]
name = "github.com/spf13/viper"
revision = "382f87b929b84ce13e9c8a375a4b217f224e6c65"

[[constraint]]
name = "golang.org/x/crypto"

[[constraint]]
name = "golang.org/x/oauth2"
revision = "442624c9ec9243441e83b374a9e22ac549b5c51d"

[[constraint]]
name = "gopkg.in/cheggaaa/pb.v1"
version = "=1.0.13"

# Get all dependencies for containers/image which is used for OCI image chaching
[[constraint]]
name = "github.com/containers/image"
revision = "8b24210344a448e58be6ed53636e2eacaa30be32"

[[constraint]]
name = "github.com/opencontainers/runc"
revision = "6b1d0e76f239ffb435445e5ae316d2676c07c6e3"

[[constraint]]
name = "github.com/docker/go-connections"
revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d"

# Use override for docker/docker and docker/distribution to ensure we get the specfied version.
# In particular we need a version which uses the altest logrus
[[override]]
name = "github.com/docker/docker"
revision = "30eb4d8cdc422b023d5f11f29a82ecb73554183b"

[[override]]
name = "github.com/docker/distribution"
revision = "5f6282db7d65e6d72ad7c2cc66310724a57be716"

[[constraint]]
name = "github.com/opencontainers/image-spec"
version = "=v1.0.0"

[[constraint]]
name = "gopkg.in/yaml.v2"
revision = "a3f3340b5840cee44f372bddb5880fcbc419b46a"

[[constraint]]
name = "github.com/pborman/uuid"
revision = "1b00554d822231195d1babd97ff4a781231955c9"

[[constraint]]
name = "golang.org/x/crypto"
revision = "453249f01cfeb54c3d549ddb75ff152ca243f9d8"

[[constraint]]
name = "github.com/sirupsen/logrus"
version = "=v1.0.0"

[[constraint]]
name = "github.com/gorilla/mux"
revision = "94e7d24fd285520f3d12ae998f7fdd6b5393d453"

[[constraint]]
name = "github.com/docker/libtrust"
revision = "aabc10ec26b754e797f9028f4589c5b7bd90dc20"

[[constraint]]
name = "github.com/ghodss/yaml"
revision = "04f313413ffd65ce25f2541bfd2b2ceec5c0908c"

[[constraint]]
name = "github.com/vbatts/tar-split"
version = "=v0.10.2"

[[constraint]]
name = "golang.org/x/sys"
revision = "43e60d72a8e2bd92ee98319ba9a384a0e9837c08"

[[constraint]]
name = "github.com/opencontainers/go-digest"
revision = "aa2ec055abd10d26d539eb630a92241b781ce4bc"
20 changes: 11 additions & 9 deletions Makefile
Expand Up @@ -44,6 +44,8 @@ VERSION_VARIABLES := -X $(REPOPATH)/pkg/version.minishiftVersion=$(MINISHIFT_VER
-X $(REPOPATH)/pkg/version.openshiftVersion=$(OPENSHIFT_VERSION) \
-X $(REPOPATH)/pkg/version.commitSha=$(COMMIT_SHA)
LDFLAGS := $(VERSION_VARIABLES) -s -w -extldflags '-static'
# Build tags atm mainly required to compile containers/image from which we only need OCI and Docker daemon transport. See issue #952
BUILD_TAGS=containers_image_openpgp containers_image_storage_stub containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay

# Setup for go-bindata to include binary assets
ADDON_ASSETS = $(CURDIR)/addons
Expand Down Expand Up @@ -78,7 +80,7 @@ __check_defined = \

.PHONY: $(GOPATH)/bin/minishift$(IS_EXE)
$(GOPATH)/bin/minishift$(IS_EXE): $(ADDON_ASSET_FILE) vendor
go install -pkgdir=$(ADDON_BINDATA_DIR) -ldflags="$(VERSION_VARIABLES)" ./cmd/minishift
go install -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) -ldflags="$(VERSION_VARIABLES)" ./cmd/minishift
vendor:
dep ensure -v

Expand All @@ -89,14 +91,14 @@ $(ADDON_ASSET_FILE): $(GOPATH)/bin/go-bindata
$(BUILD_DIR)/$(GOOS)-$(GOARCH):
mkdir -p $(BUILD_DIR)/$(GOOS)-$(GOARCH)

$(BUILD_DIR)/darwin-amd64/minishift: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=darwin go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/darwin-amd64/minishift ./cmd/minishift
$(BUILD_DIR)/darwin-amd64/minishift: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=darwin go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/darwin-amd64/minishift ./cmd/minishift

$(BUILD_DIR)/linux-amd64/minishift: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/linux-amd64/minishift ./cmd/minishift
$(BUILD_DIR)/linux-amd64/minishift: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/linux-amd64/minishift ./cmd/minishift

$(BUILD_DIR)/windows-amd64/minishift.exe: vendor $(ADDON_ASSET_FILE) $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=windows go build -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/windows-amd64/minishift.exe ./cmd/minishift
$(BUILD_DIR)/windows-amd64/minishift.exe: $(ADDON_ASSET_FILE) vendor $(BUILD_DIR)/$(GOOS)-$(GOARCH)
CGO_ENABLED=0 GOARCH=amd64 GOOS=windows go build -tags "$(BUILD_TAGS)" -pkgdir=$(ADDON_BINDATA_DIR) --installsuffix cgo -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/windows-amd64/minishift.exe ./cmd/minishift

$(GOPATH)/bin/gh-release:
go get -u github.com/progrium/gh-release/...
Expand Down Expand Up @@ -137,7 +139,7 @@ link_check_docs: gen_docs

$(DOCS_SYNOPISIS_DIR)/*.md: vendor $(ADDON_ASSET_FILE)
@# https://github.com/golang/go/issues/15038#issuecomment-207631885 ( CGO_ENABLED=0 )
DOCS_SYNOPISIS_DIR=$(DOCS_SYNOPISIS_DIR) CGO_ENABLED=0 go run -ldflags="$(LDFLAGS)" -tags gendocs gen_help_text.go
DOCS_SYNOPISIS_DIR=$(DOCS_SYNOPISIS_DIR) CGO_ENABLED=0 go run -tags "$(BUILD_TAGS) gendocs" -ldflags="$(LDFLAGS)" gen_help_text.go

.PHONY: synopsis_docs
synopsis_docs: $(DOCS_SYNOPISIS_DIR)/*.md
Expand Down Expand Up @@ -193,7 +195,7 @@ clean:

.PHONY: test
test: vendor $(ADDON_ASSET_FILE)
@go test -ldflags="$(VERSION_VARIABLES)" -v $(shell $(PACKAGES))
@go test -tags "$(BUILD_TAGS)" -ldflags="$(VERSION_VARIABLES)" -v $(shell $(PACKAGES))

.PHONY: integration
integration: GODOG_OPTS = --tags=basic
Expand Down
12 changes: 8 additions & 4 deletions cmd/minishift/cmd/config/config.go
Expand Up @@ -95,9 +95,11 @@ var (
HostFoldersMountPath = createConfigSetting("hostfolders-mountpath", SetString, nil, nil, true)
HostFoldersAutoMount = createConfigSetting("hostfolders-automount", SetBool, nil, nil, true)

// Image caching
ImageCaching = createConfigSetting("image-caching", SetBool, nil, nil, true)
CacheImages = createConfigSetting("cache-images", SetSlice, nil, nil, true)

// Preflight checks (before start)
// Pre-flight checks (before start)
SkipCheckKVMDriver = createConfigSetting("skip-check-kvm-driver", SetBool, nil, nil, true)
WarnCheckKVMDriver = createConfigSetting("warn-check-kvm-driver", SetBool, nil, nil, true)
SkipCheckXHyveDriver = createConfigSetting("skip-check-xhyve-driver", SetBool, nil, nil, true)
Expand All @@ -108,7 +110,8 @@ var (
WarnCheckIsoUrl = createConfigSetting("warn-check-iso-url", SetBool, nil, nil, true)
SkipCheckVMDriver = createConfigSetting("skip-check-vm-driver", SetBool, nil, nil, true)
WarnCheckVMDriver = createConfigSetting("warn-check-vm-driver", SetBool, nil, nil, true)
// Preflight checks (after start)

// Pre-flight checks (after start)
SkipInstanceIP = createConfigSetting("skip-check-instance-ip", SetBool, nil, nil, true)
WarnInstanceIP = createConfigSetting("warn-check-instance-ip", SetBool, nil, nil, true)
SkipCheckNetworkHost = createConfigSetting("skip-check-network-host", SetBool, nil, nil, true)
Expand All @@ -121,7 +124,8 @@ var (
WarnCheckStorageMount = createConfigSetting("warn-check-storage-mount", SetBool, nil, nil, true)
SkipCheckStorageUsage = createConfigSetting("skip-check-storage-usage", SetBool, nil, nil, true)
WarnCheckStorageUsage = createConfigSetting("warn-check-storage-usage", SetBool, nil, nil, true)
// Preflight values

// Pre-flight values
CheckNetworkHttpHost = createConfigSetting("check-network-http-host", SetString, nil, nil, true)
CheckNetworkPingHost = createConfigSetting("check-network-ping-host", SetString, nil, nil, true)

Expand All @@ -130,7 +134,7 @@ var (
IPAddress = createConfigSetting("network-ipaddress", SetString, []setFn{validations.IsValidIPv4Address}, nil, true)
Netmask = createConfigSetting("network-netmask", SetString, []setFn{validations.IsValidNetmask}, nil, true)
Gateway = createConfigSetting("network-gateway", SetString, []setFn{validations.IsValidIPv4Address}, nil, true)
Nameserver = createConfigSetting("network-nameserver", SetString, []setFn{validations.IsValidIPv4Address}, nil, true)
NameServer = createConfigSetting("network-nameserver", SetString, []setFn{validations.IsValidIPv4Address}, nil, true)
)

func createConfigSetting(name string, set func(MinishiftConfig, string, string) error, validations []setFn, callbacks []setFn, isApply bool) *Setting {
Expand Down
1 change: 1 addition & 0 deletions cmd/minishift/cmd/config/view.go
Expand Up @@ -59,6 +59,7 @@ var configViewCmd = &cobra.Command{

func init() {
excludedConfigKeys["addons"] = true
excludedConfigKeys["cache-images"] = true
configViewCmd.Flags().StringVar(&configViewFormat, "format", DefaultConfigViewFormat,
`Go template format to apply to the configuration file. For more information about Go templates, see: https://golang.org/pkg/text/template/
For the list of configurable variables for the template, see the struct values section of ConfigViewTemplate at: https://godoc.org/github.com/minishift/minishift/cmd/minishift/cmd/config#ConfigViewTemplate`)
Expand Down
38 changes: 38 additions & 0 deletions cmd/minishift/cmd/image/cache_config.go
@@ -0,0 +1,38 @@
/*
Copyright (C) 2017 Red Hat, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package image

import (
"github.com/spf13/cobra"
)

const (
noImageSpecified = "You need to specify one or more images."
)

var ImageCacheConfigCmd = &cobra.Command{
Use: "cache-config SUBCOMMAND [flags]",
Short: "Controls the list of cached images which are implicitly imported and exported.",
Long: "Controls the list of cached images which are implicitly imported and exported.",
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
},
}

func init() {
ImageCmd.AddCommand(ImageCacheConfigCmd)
}
62 changes: 62 additions & 0 deletions cmd/minishift/cmd/image/cache_config_add.go
@@ -0,0 +1,62 @@
/*
Copyright (C) 2017 Red Hat, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package image

import (
"github.com/spf13/cobra"

"fmt"
"github.com/minishift/minishift/cmd/minishift/cmd/config"
"github.com/minishift/minishift/pkg/util/os/atexit"
"github.com/minishift/minishift/pkg/util/strings"
)

var (
addConfiguredImageCmd = &cobra.Command{
Use: "add [image ...]",
Short: "Adds the specified images to the list of configured images for import and export.",
Long: "Adds the specified images to the list of configured images for import and export.",
Run: addConfiguredImage,
}
)

func addConfiguredImage(cmd *cobra.Command, args []string) {
if len(args) == 0 {
atexit.ExitWithMessage(1, noImageSpecified)
}

normalizedImageNames, err := normalizeImageNames(args)
if err != nil {
atexit.ExitWithMessage(1, fmt.Sprintf("Invalid image name: %v", err))
}

minishiftConfig := getMinishiftConfig()
cacheImages := getConfiguredCachedImages(minishiftConfig)

for _, image := range normalizedImageNames {
if !strings.Contains(cacheImages, image) {
cacheImages = append(cacheImages, image)
}
}

minishiftConfig[config.CacheImages.Name] = cacheImages
config.WriteConfig(minishiftConfig)
}

func init() {
ImageCacheConfigCmd.AddCommand(addConfiguredImageCmd)
}

0 comments on commit 52a24d7

Please sign in to comment.