-
Notifications
You must be signed in to change notification settings - Fork 561
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
store: download deltas if explicitly enabled #2017
Changes from all commits
3c6c8cc
47fcc03
b1d5adb
4b6d3f0
975a477
1419b6c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ package store | |
import ( | ||
"bytes" | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
"io" | ||
"io/ioutil" | ||
|
@@ -141,7 +142,7 @@ type Config struct { | |
Series string | ||
|
||
DetailFields []string | ||
DeltaFormats []string | ||
DeltaFormat string | ||
} | ||
|
||
// Store represents the ubuntu snap store | ||
|
@@ -160,7 +161,7 @@ type Store struct { | |
fallbackStoreID string | ||
|
||
detailFields []string | ||
deltaFormats []string | ||
deltaFormat string | ||
// reused http client | ||
client *http.Client | ||
|
||
|
@@ -312,8 +313,8 @@ type searchResults struct { | |
// The fields we are interested in | ||
var detailFields = getStructFields(snapDetails{}) | ||
|
||
// The default delta formats if none are configured. | ||
var defaultSupportedDeltaFormats = []string{"xdelta"} | ||
// The default delta format if not configured. | ||
var defaultSupportedDeltaFormat = "xdelta" | ||
|
||
// New creates a new Store with the given access configuration and for given the store id. | ||
func New(cfg *Config, authContext auth.AuthContext) *Store { | ||
|
@@ -357,9 +358,9 @@ func New(cfg *Config, authContext auth.AuthContext) *Store { | |
series = cfg.Series | ||
} | ||
|
||
deltaFormats := cfg.DeltaFormats | ||
if deltaFormats == nil { | ||
deltaFormats = defaultSupportedDeltaFormats | ||
deltaFormat := cfg.DeltaFormat | ||
if deltaFormat == "" { | ||
deltaFormat = defaultSupportedDeltaFormat | ||
} | ||
|
||
// see https://wiki.ubuntu.com/AppStore/Interfaces/ClickPackageIndex | ||
|
@@ -377,7 +378,7 @@ func New(cfg *Config, authContext auth.AuthContext) *Store { | |
detailFields: fields, | ||
client: newHTTPClient(), | ||
authContext: authContext, | ||
deltaFormats: deltaFormats, | ||
deltaFormat: deltaFormat, | ||
} | ||
} | ||
|
||
|
@@ -1060,9 +1061,9 @@ func (s *Store) ListRefresh(installed []*RefreshCandidate, user *auth.UserState) | |
Data: jsonData, | ||
} | ||
|
||
if os.Getenv("SNAPPY_USE_DELTAS") == "1" { | ||
if os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL") == "1" { | ||
reqOptions.ExtraHeaders = map[string]string{ | ||
"X-Ubuntu-Delta-Formats": strings.Join(s.deltaFormats, ","), | ||
"X-Ubuntu-Delta-Formats": s.deltaFormat, | ||
} | ||
} | ||
|
||
|
@@ -1132,6 +1133,25 @@ func (s *Store) Download(name string, downloadInfo *snap.DownloadInfo, pbar prog | |
} | ||
}() | ||
|
||
if os.Getenv("SNAPD_USE_DELTAS_EXPERIMENTAL") == "1" && len(downloadInfo.Deltas) == 1 { | ||
downloadDir, err := ioutil.TempDir("", name+"-deltas") | ||
if err == nil { | ||
defer os.RemoveAll(downloadDir) | ||
|
||
deltaPath, err := s.downloadDelta(name, downloadDir, downloadInfo, pbar, user) | ||
// We revert to normal downloads if there is any error | ||
if err != nil { | ||
// Just log the error and continue with the normal non-delta | ||
// download. | ||
logger.Noticef("Cannot download deltas for %s: %v", name, err) | ||
} else { | ||
// Currently even on successful delta downloads, continue with the | ||
// normal full download. | ||
logger.Debugf("Successfully downloaded deltas for %s at %s", name, deltaPath) | ||
} | ||
} | ||
} | ||
|
||
url := downloadInfo.AnonDownloadURL | ||
if url == "" || user != nil { | ||
url = downloadInfo.DownloadURL | ||
|
@@ -1195,6 +1215,49 @@ var download = func(name, downloadURL string, user *auth.UserState, s *Store, w | |
return err | ||
} | ||
|
||
// downloadDelta downloads the delta for the preferred format, returning the path. | ||
func (s *Store) downloadDelta(name string, downloadDir string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState) (string, error) { | ||
|
||
if len(downloadInfo.Deltas) != 1 { | ||
return "", errors.New("store returned more than one download delta") | ||
} | ||
|
||
deltaInfo := downloadInfo.Deltas[0] | ||
|
||
if deltaInfo.Format != s.deltaFormat { | ||
return "", fmt.Errorf("store returned a download delta with the wrong format (%q instead of the configured %s format)", deltaInfo.Format, s.deltaFormat) | ||
} | ||
|
||
deltaName := fmt.Sprintf("%s_%d_%d_delta.%s", name, deltaInfo.FromRevision, deltaInfo.ToRevision, deltaInfo.Format) | ||
|
||
w, err := os.Create(path.Join(downloadDir, deltaName)) | ||
if err != nil { | ||
return "", err | ||
} | ||
deltaPath := w.Name() | ||
defer func() { | ||
if cerr := w.Close(); cerr != nil && err == nil { | ||
err = cerr | ||
} | ||
if err != nil { | ||
os.Remove(w.Name()) | ||
deltaPath = "" | ||
} | ||
}() | ||
|
||
url := deltaInfo.AnonDownloadURL | ||
if url == "" || user != nil { | ||
url = deltaInfo.DownloadURL | ||
} | ||
|
||
err = download(deltaName, url, user, s, w, pbar) | ||
if err != nil { | ||
return "", err | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaking an open file here, without the defer above. |
||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. so at this point the file is downloaded and stored somewhere, but you have no guarantees that it ever made it to disc. This treatment is fine for tempfiles (such as what Download does) but not at all fine for things you expect to be on permanent storage. You need to download to a temporary filename (in the same directory you want it to then live), and on success sync the file, rename it, and sync the directory. Take a look at AtomicWriteFileChown in osutil/io.go. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We chatted about this - and you mentioned that it is fine without a w.Sync() even, as xdelta doesn't go through raw io, but goes through the vfs. |
||
return deltaPath, nil | ||
} | ||
|
||
type assertionSvcError struct { | ||
Status int `json:"status"` | ||
Type string `json:"type"` | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
defer w.Close()
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done... although I did the similar defer func(){} to ensure we check the error on w.Close() and return that error unless there's an error being returned already (and either way deleting the file in that case). Let me know if that's not what you wanted.