store: download deltas if explicitly enabled

snap/info.go

@@ -234,6 +234,10 @@ type DownloadInfo struct {
 	Size     int64  `json:"size,omitempty"`
 	Sha3_384 string `json:"sha3-384,omitempty"`
 
+	// The server can include information about available deltas for a given
+	// snap at a specific revision during refresh. Currently during refresh the
+	// server will provide single matching deltas only, from the clients
+	// revision to the target revision when available, per requested format.
 	Deltas []DeltaInfo `json:"deltas,omitempty"`
 }
 

store/store.go

@@ -23,6 +23,7 @@ package store
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -141,7 +142,7 @@ type Config struct {
 	Series       string
 
 	DetailFields []string
-	DeltaFormats []string
+	DeltaFormat  string
 }
 
 // Store represents the ubuntu snap store
@@ -160,7 +161,7 @@ type Store struct {
 	fallbackStoreID string
 
 	detailFields []string
-	deltaFormats []string
+	deltaFormat  string
 	// reused http client
 	client *http.Client
 
@@ -312,8 +313,8 @@ type searchResults struct {
 // The fields we are interested in
 var detailFields = getStructFields(snapDetails{})
 
-// The default delta formats if none are configured.
-var defaultSupportedDeltaFormats = []string{"xdelta"}
+// The default delta format if not configured.
+var defaultSupportedDeltaFormat = "xdelta"
 
 // New creates a new Store with the given access configuration and for given the store id.
 func New(cfg *Config, authContext auth.AuthContext) *Store {
@@ -357,9 +358,9 @@ func New(cfg *Config, authContext auth.AuthContext) *Store {
 		series = cfg.Series
 	}
 
-	deltaFormats := cfg.DeltaFormats
-	if deltaFormats == nil {
-		deltaFormats = defaultSupportedDeltaFormats
+	deltaFormat := cfg.DeltaFormat
+	if deltaFormat == "" {
+		deltaFormat = defaultSupportedDeltaFormat
 	}
 
 	// see https://wiki.ubuntu.com/AppStore/Interfaces/ClickPackageIndex
@@ -377,7 +378,7 @@ func New(cfg *Config, authContext auth.AuthContext) *Store {
 		detailFields:      fields,
 		client:            newHTTPClient(),
 		authContext:       authContext,
-		deltaFormats:      deltaFormats,
+		deltaFormat:       deltaFormat,
 	}
 }
 
@@ -1060,9 +1061,9 @@ func (s *Store) ListRefresh(installed []*RefreshCandidate, user *auth.UserState)
 		Data:        jsonData,
 	}
 
-	if os.Getenv("SNAPPY_USE_DELTAS") == "1" {
+	if os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL") == "1" {
 		reqOptions.ExtraHeaders = map[string]string{
-			"X-Ubuntu-Delta-Formats": strings.Join(s.deltaFormats, ","),
+			"X-Ubuntu-Delta-Formats": s.deltaFormat,
 		}
 	}
 
@@ -1132,6 +1133,25 @@ func (s *Store) Download(name string, downloadInfo *snap.DownloadInfo, pbar prog
 		}
 	}()
 
+	if os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL") == "1" && len(downloadInfo.Deltas) == 1 {
+		downloadDir, err := ioutil.TempDir("", name+"-deltas")
+		if err == nil {
+			defer os.RemoveAll(downloadDir)
+
+			deltaPath, err := s.downloadDelta(name, downloadDir, downloadInfo, pbar, user)
+			// We revert to normal downloads if there is any error
+			if err != nil {
+				// Just log the error and continue with the normal non-delta
+				// download.
+				logger.Noticef("Cannot download deltas for %s: %v", name, err)
+			} else {
+				// Currently even on successful delta downloads, continue with the
+				// normal full download.
+				logger.Debugf("Successfully downloaded deltas for %s at %s", name, deltaPath)
+			}
+		}
+	}
+
 	url := downloadInfo.AnonDownloadURL
 	if url == "" || user != nil {
 		url = downloadInfo.DownloadURL
@@ -1195,6 +1215,49 @@ var download = func(name, downloadURL string, user *auth.UserState, s *Store, w
 	return err
 }
 
+// downloadDelta downloads the delta for the preferred format, returning the path.
+func (s *Store) downloadDelta(name string, downloadDir string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) (string, error) {
+
+	if len(downloadInfo.Deltas) != 1 {
+		return "", errors.New("store returned more than one download delta")
+	}
+
+	deltaInfo := downloadInfo.Deltas[0]
+
+	if deltaInfo.Format != s.deltaFormat {
+		return "", fmt.Errorf("store returned a download delta with the wrong format (%q instead of the configured %s format)", deltaInfo.Format, s.deltaFormat)
+	}
+
+	deltaName := fmt.Sprintf("%s_%d_%d_delta.%s", name, deltaInfo.FromRevision, deltaInfo.ToRevision, deltaInfo.Format)
+
+	w, err := os.Create(path.Join(downloadDir, deltaName))
+	if err != nil {
+		return "", err
+	}
+	deltaPath := w.Name()
+	defer func() {
+		if cerr := w.Close(); cerr != nil && err == nil {
+			err = cerr
+		}
+		if err != nil {
+			os.Remove(w.Name())
+			deltaPath = ""
+		}
+	}()
+
+	url := deltaInfo.AnonDownloadURL
+	if url == "" || user != nil {
+		url = deltaInfo.DownloadURL
+	}
+
+	err = download(deltaName, url, user, s, w, pbar)
+	if err != nil {
+		return "", err
+	}
+
+	return deltaPath, nil
+}
+
 type assertionSvcError struct {
 	Status int    `json:"status"`
 	Type   string `json:"type"`

store/store_test.go

@@ -22,13 +22,15 @@ package store
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path"
 	"strings"
 	"testing"
 
@@ -391,6 +393,181 @@ func (t *remoteRepoTestSuite) TestActualDownload500(c *C) {
 	c.Check(n, Equals, len(downloadBackoffs)) // woo!!
 }
 
+type downloadBehaviour []struct {
+	url   string
+	error bool
+}
+
+var deltaTests = []struct {
+	downloads       downloadBehaviour
+	info            snap.DownloadInfo
+	expectedContent string
+}{{
+	// The full snap is downloaded after the delta (currently,
+	// until the code applying the delta lands).
+	downloads: downloadBehaviour{
+		{url: "delta-url"},
+		{url: "full-snap-url"},
+	},
+	info: snap.DownloadInfo{
+		AnonDownloadURL: "full-snap-url",
+		Deltas: []snap.DeltaInfo{
+			{AnonDownloadURL: "delta-url", Format: "xdelta"},
+		},
+	},
+	expectedContent: "full-snap-url-content",
+}, {
+	// If there is an error during the delta download, the
+	// full snap is downloaded as per normal.
+	downloads: downloadBehaviour{
+		{error: true},
+		{url: "full-snap-url"},
+	},
+	info: snap.DownloadInfo{
+		AnonDownloadURL: "full-snap-url",
+		Deltas: []snap.DeltaInfo{
+			{AnonDownloadURL: "delta-url", Format: "xdelta"},
+		},
+	},
+	expectedContent: "full-snap-url-content",
+}, {
+	// If more than one matching delta is returned by the store
+	// we ignore deltas and do the full download.
+	downloads: downloadBehaviour{
+		{url: "full-snap-url"},
+	},
+	info: snap.DownloadInfo{
+		AnonDownloadURL: "full-snap-url",
+		Deltas: []snap.DeltaInfo{
+			{AnonDownloadURL: "delta-url", Format: "xdelta"},
+			{AnonDownloadURL: "delta-url-2", Format: "xdelta"},
+		},
+	},
+	expectedContent: "full-snap-url-content",
+}}
+
+func (t *remoteRepoTestSuite) TestDownloadWithDeltas(c *C) {
+	origUseDeltas := os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL")
+	defer os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", origUseDeltas)
+	c.Assert(os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", "1"), IsNil)
+
+	for _, testCase := range deltaTests {
+
+		downloadIndex := 0
+		download = func(name, url string, user *auth.UserState, s *Store, w io.Writer, pbar progress.Meter) error {
+			if testCase.downloads[downloadIndex].error {
+				downloadIndex++
+				return errors.New("Bang")
+			}
+			c.Check(url, Equals, testCase.downloads[downloadIndex].url)
+			w.Write([]byte(testCase.downloads[downloadIndex].url + "-content"))
+			downloadIndex++
+			return nil
+		}
+
+		path, err := t.store.Download("foo", &testCase.info, nil, nil)
+
+		c.Assert(err, IsNil)
+		defer os.Remove(path)
+		content, err := ioutil.ReadFile(path)
+		c.Assert(err, IsNil)
+		c.Assert(string(content), Equals, testCase.expectedContent)
+	}
+}
+
+var downloadDeltaTests = []struct {
+	info          snap.DownloadInfo
+	authenticated bool
+	format        string
+	expectedURL   string
+	expectedPath  string
+}{{
+	// An unauthenticated request downloads the anonymous delta url.
+	info: snap.DownloadInfo{
+		Deltas: []snap.DeltaInfo{
+			{AnonDownloadURL: "anon-delta-url", Format: "xdelta", FromRevision: 24, ToRevision: 26},
+		},
+	},
+	authenticated: false,
+	format:        "xdelta",
+	expectedURL:   "anon-delta-url",
+	expectedPath:  "snapname_24_26_delta.xdelta",
+}, {
+	// An authenticated request downloads the authenticated delta url.
+	info: snap.DownloadInfo{
+		Deltas: []snap.DeltaInfo{
+			{DownloadURL: "auth-delta-url", Format: "xdelta", FromRevision: 24, ToRevision: 26},
+		},
+	},
+	authenticated: true,
+	format:        "xdelta",
+	expectedURL:   "auth-delta-url",
+	expectedPath:  "snapname_24_26_delta.xdelta",
+}, {
+	// An error is returned if more than one matching delta is returned by the store,
+	// though this may be handled in the future.
+	info: snap.DownloadInfo{
+		Deltas: []snap.DeltaInfo{
+			{DownloadURL: "xdelta-delta-url", Format: "xdelta", FromRevision: 24, ToRevision: 25},
+			{DownloadURL: "bsdiff-delta-url", Format: "xdelta", FromRevision: 25, ToRevision: 26},
+		},
+	},
+	authenticated: false,
+	format:        "xdelta",
+	expectedURL:   "",
+	expectedPath:  "",
+}, {
+	// If the supported format isn't available, an error is returned.
+	info: snap.DownloadInfo{
+		Deltas: []snap.DeltaInfo{
+			{DownloadURL: "xdelta-delta-url", Format: "xdelta", FromRevision: 24, ToRevision: 26},
+			{DownloadURL: "ydelta-delta-url", Format: "ydelta", FromRevision: 24, ToRevision: 26},
+		},
+	},
+	authenticated: false,
+	format:        "bsdiff",
+	expectedURL:   "",
+	expectedPath:  "",
+}}
+
+func (t *remoteRepoTestSuite) TestDownloadDelta(c *C) {
+	origUseDeltas := os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL")
+	defer os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", origUseDeltas)
+	c.Assert(os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", "1"), IsNil)
+
+	for _, testCase := range downloadDeltaTests {
+		t.store.deltaFormat = testCase.format
+		download = func(name, url string, user *auth.UserState, s *Store, w io.Writer, pbar progress.Meter) error {
+			expectedUser := t.user
+			if !testCase.authenticated {
+				expectedUser = nil
+			}
+			c.Check(user, Equals, expectedUser)
+			c.Check(url, Equals, testCase.expectedURL)
+			return nil
+		}
+
+		downloadDir, err := ioutil.TempDir("", "")
+		c.Assert(err, IsNil)
+		defer os.RemoveAll(downloadDir)
+
+		authedUser := t.user
+		if !testCase.authenticated {
+			authedUser = nil
+		}
+
+		deltaPath, err := t.store.downloadDelta("snapname", downloadDir, &testCase.info, nil, authedUser)
+
+		if testCase.expectedPath == "" {
+			c.Assert(deltaPath, Equals, "")
+			c.Assert(err, NotNil)
+		} else {
+			c.Assert(err, IsNil)
+			c.Assert(deltaPath, Equals, path.Join(downloadDir, testCase.expectedPath))
+		}
+	}
+}
+
 func (t *remoteRepoTestSuite) TestDoRequestSetsAuth(c *C) {
 	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		c.Check(r.UserAgent(), Equals, userAgent)
@@ -1668,9 +1845,9 @@ var MockUpdatesWithDeltasJSON = `
 `
 
 func (t *remoteRepoTestSuite) TestUbuntuStoreRepositoryListRefreshWithDeltas(c *C) {
-	orig_use_deltas := os.Getenv("SNAPPY_USE_DELTAS")
-	defer os.Setenv("SNAPPY_USE_DELTAS", orig_use_deltas)
-	c.Assert(os.Setenv("SNAPPY_USE_DELTAS", "1"), IsNil)
+	origUseDeltas := os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL")
+	defer os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", origUseDeltas)
+	c.Assert(os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", "1"), IsNil)
 
 	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		c.Check(r.Header.Get("X-Ubuntu-Delta-Formats"), Equals, `xdelta`)
@@ -1743,9 +1920,9 @@ func (t *remoteRepoTestSuite) TestUbuntuStoreRepositoryListRefreshWithDeltas(c *
 
 func (t *remoteRepoTestSuite) TestUbuntuStoreRepositoryListRefreshWithoutDeltas(c *C) {
 	// Verify the X-Delta-Format header is not set.
-	orig_use_deltas := os.Getenv("SNAPPY_USE_DELTAS")
-	defer os.Setenv("SNAPPY_USE_DELTAS", orig_use_deltas)
-	c.Assert(os.Setenv("SNAPPY_USE_DELTAS", "0"), IsNil)
+	origUseDeltas := os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL")
+	defer os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", origUseDeltas)
+	c.Assert(os.Setenv("SNAPD_USE_DELTAS_EXPERIMENTAL", "0"), IsNil)
 
 	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		c.Check(r.Header.Get("X-Ubuntu-Delta-Formats"), Equals, ``)