Skip to content

Commit

Permalink
Merge pull request #13393 from hpidcock/fix-caas-iaas-tools
Browse files Browse the repository at this point in the history 
#13393

- Fix tool downloads for IAAS hosted model on CAAS
- Fixes snap LXD config path

## QA steps

- Bootstrap microk8s
- `lxc remote add lxdcloud my.ip.add.ress`
- `juju autoload-credentials`
- `juju add-model lxdcloud lxdcloud`
- `juju deploy apache2` (might need a custom simplestream or manually edit version/version.go to a previous version)
- Machine should start and charm should become ready

## Documentation changes

N/A

## Bug reference

https://bugs.launchpad.net/juju/+bug/1943265
  • Loading branch information
@jujubot
jujubot committed Oct 7, 2021
2 parents 3d09733 + cdcb109 commit 9983864
Show file tree
Hide file tree
Showing 7 changed files with 169 additions and 63 deletions.
39 changes: 30 additions & 9 deletions apiserver/tools.go
View file
Expand Up @@ -229,7 +229,7 @@ func (h *toolsDownloadHandler) getToolsForRequest(r *http.Request, st *state.Sta
if osTypeName != "" {
storageVers.Release = osTypeName
}
err = h.fetchAndCacheTools(vers, storageVers)
err = h.fetchAndCacheTools(vers, storageVers, st, storage)
if err != nil {
err = errors.Annotate(err, "error fetching agent binaries")
} else {
Expand All @@ -249,15 +249,42 @@ func (h *toolsDownloadHandler) getToolsForRequest(r *http.Request, st *state.Sta
func (h *toolsDownloadHandler) fetchAndCacheTools(
v version.Binary,
storageVers version.Binary,
st *state.State,
modelStorage binarystorage.Storage,
) error {
systemState := h.ctxt.statePool().SystemState()

controllerModel, err := systemState.Model()
if err != nil {
return err
}

var model *state.Model
var storage binarystorage.Storage
switch controllerModel.Type() {
case state.ModelTypeCAAS:
// TODO(caas): unify tool fetching
// Cache the tools against the model when the controller is CAAS.
model, err = st.Model()
if err != nil {
return err
}
storage = modelStorage
case state.ModelTypeIAAS:
// Cache the tools against the controller when the controller is IAAS.
model = controllerModel
controllerStorage, err := systemState.ToolsStorage()
if err != nil {
return err
}
defer controllerStorage.Close()
storage = controllerStorage
default:
return errors.NotValidf("model type %q", controllerModel.Type())
}

newEnviron := stateenvirons.GetNewEnvironFunc(environs.New)
env, err := newEnviron(controllerModel)
env, err := newEnviron(model)
if err != nil {
return err
}
Expand All @@ -284,12 +311,6 @@ func (h *toolsDownloadHandler) fetchAndCacheTools(
return errors.New(msg)
}

controllerStorage, err := systemState.ToolsStorage()
if err != nil {
return err
}
defer controllerStorage.Close()

data, respSha256, size, err := tmpCacheAndHash(resp.Body)
if err != nil {
return err
Expand All @@ -307,7 +328,7 @@ func (h *toolsDownloadHandler) fetchAndCacheTools(
Size: exactTools.Size,
SHA256: exactTools.SHA256,
}
if err := controllerStorage.Add(data, md); err != nil {
if err := storage.Add(data, md); err != nil {
return errors.Annotate(err, "error caching agent binaries")
}

Expand Down
159 changes: 109 additions & 50 deletions apiserver/tools_test.go
View file
Expand Up @@ -38,30 +38,28 @@ import (
coretools "github.com/juju/juju/tools"
)

type toolsSuite struct {
type baseToolsSuite struct {
apiserverBaseSuite
}

var _ = gc.Suite(&toolsSuite{})

func (s *toolsSuite) toolsURL(query string) *url.URL {
func (s *baseToolsSuite) toolsURL(query string) *url.URL {
return s.modelToolsURL(s.Model.UUID(), query)
}

func (s *toolsSuite) modelToolsURL(model, query string) *url.URL {
func (s *baseToolsSuite) modelToolsURL(model, query string) *url.URL {
u := s.URL(fmt.Sprintf("/model/%s/tools", model), nil)
u.RawQuery = query
return u
}

func (s *toolsSuite) toolsURI(query string) string {
func (s *baseToolsSuite) toolsURI(query string) string {
if query != "" && query[0] == '?' {
query = query[1:]
}
return s.toolsURL(query).String()
}

func (s *toolsSuite) uploadRequest(c *gc.C, url, contentType string, content io.Reader) *http.Response {
func (s *baseToolsSuite) uploadRequest(c *gc.C, url, contentType string, content io.Reader) *http.Response {
return s.sendHTTPRequest(c, apitesting.HTTPRequestParams{
Method: "POST",
URL: url,
Expand All @@ -70,7 +68,7 @@ func (s *toolsSuite) uploadRequest(c *gc.C, url, contentType string, content io.
})
}

func (s *toolsSuite) downloadRequest(c *gc.C, version version.Binary, uuid string) *http.Response {
func (s *baseToolsSuite) downloadRequest(c *gc.C, version version.Binary, uuid string) *http.Response {
url := s.toolsURL("")
if uuid == "" {
url.Path = fmt.Sprintf("/tools/%s", version)
Expand All @@ -80,32 +78,85 @@ func (s *toolsSuite) downloadRequest(c *gc.C, version version.Binary, uuid strin
return apitesting.SendHTTPRequest(c, apitesting.HTTPRequestParams{Method: "GET", URL: url.String()})
}

func (s *toolsSuite) assertUploadResponse(c *gc.C, resp *http.Response, agentTools *coretools.Tools) {
func (s *baseToolsSuite) assertUploadResponse(c *gc.C, resp *http.Response, agentTools *coretools.Tools) {
toolsResponse := s.assertResponse(c, resp, http.StatusOK)
c.Check(toolsResponse.Error, gc.IsNil)
c.Check(toolsResponse.ToolsList, jc.DeepEquals, coretools.List{agentTools})
}

func (s *toolsSuite) assertJSONErrorResponse(c *gc.C, resp *http.Response, expCode int, expError string) {
func (s *baseToolsSuite) assertJSONErrorResponse(c *gc.C, resp *http.Response, expCode int, expError string) {
toolsResponse := s.assertResponse(c, resp, expCode)
c.Check(toolsResponse.ToolsList, gc.IsNil)
c.Check(toolsResponse.Error, gc.NotNil)
c.Check(toolsResponse.Error.Message, gc.Matches, expError)
}

func (s *toolsSuite) assertPlainErrorResponse(c *gc.C, resp *http.Response, expCode int, expError string) {
func (s *baseToolsSuite) assertPlainErrorResponse(c *gc.C, resp *http.Response, expCode int, expError string) {
body := apitesting.AssertResponse(c, resp, expCode, "text/plain; charset=utf-8")
c.Assert(string(body), gc.Matches, expError+"\n")
}

func (s *toolsSuite) assertResponse(c *gc.C, resp *http.Response, expStatus int) params.ToolsResult {
func (s *baseToolsSuite) assertResponse(c *gc.C, resp *http.Response, expStatus int) params.ToolsResult {
body := apitesting.AssertResponse(c, resp, expStatus, params.ContentTypeJSON)
var toolsResponse params.ToolsResult
err := json.Unmarshal(body, &toolsResponse)
c.Assert(err, jc.ErrorIsNil, gc.Commentf("Body: %s", body))
return toolsResponse
}

func (s *baseToolsSuite) storeFakeTools(c *gc.C, st *state.State, content string, metadata binarystorage.Metadata) *coretools.Tools {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
err = storage.Add(strings.NewReader(content), metadata)
c.Assert(err, jc.ErrorIsNil)
return &coretools.Tools{
Version: version.MustParseBinary(metadata.Version),
Size: metadata.Size,
SHA256: metadata.SHA256,
}
}

func (s *baseToolsSuite) getToolsFromStorage(c *gc.C, st *state.State, vers string) (binarystorage.Metadata, []byte) {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
metadata, r, err := storage.Open(vers)
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
r.Close()
c.Assert(err, jc.ErrorIsNil)
return metadata, data
}

func (s *baseToolsSuite) getToolsMetadataFromStorage(c *gc.C, st *state.State) []binarystorage.Metadata {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
metadata, err := storage.AllMetadata()
c.Assert(err, jc.ErrorIsNil)
return metadata
}

func (s *baseToolsSuite) testDownload(c *gc.C, tools *coretools.Tools, uuid string) []byte {
resp := s.downloadRequest(c, tools.Version, uuid)
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
c.Assert(err, jc.ErrorIsNil)
c.Assert(data, gc.HasLen, int(tools.Size))

hash := sha256.New()
hash.Write(data)
c.Assert(fmt.Sprintf("%x", hash.Sum(nil)), gc.Equals, tools.SHA256)
return data
}

type toolsSuite struct {
baseToolsSuite
}

var _ = gc.Suite(&toolsSuite{})

func (s *toolsSuite) TestToolsUploadedSecurely(c *gc.C) {
url := s.toolsURL("")
url.Scheme = "http"
Expand Down Expand Up @@ -505,49 +556,57 @@ func (s *toolsSuite) TestDownloadMissingConcurrent(c *gc.C) {
c.Assert(resolutions, gc.Equals, len(toolsBinaries))
}

func (s *toolsSuite) storeFakeTools(c *gc.C, st *state.State, content string, metadata binarystorage.Metadata) *coretools.Tools {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
err = storage.Add(strings.NewReader(content), metadata)
c.Assert(err, jc.ErrorIsNil)
return &coretools.Tools{
Version: version.MustParseBinary(metadata.Version),
Size: metadata.Size,
SHA256: metadata.SHA256,
}
type caasToolsSuite struct {
baseToolsSuite
}

func (s *toolsSuite) getToolsFromStorage(c *gc.C, st *state.State, vers string) (binarystorage.Metadata, []byte) {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
metadata, r, err := storage.Open(vers)
c.Assert(err, jc.ErrorIsNil)
data, err := ioutil.ReadAll(r)
r.Close()
c.Assert(err, jc.ErrorIsNil)
return metadata, data
}
var _ = gc.Suite(&caasToolsSuite{})

func (s *toolsSuite) getToolsMetadataFromStorage(c *gc.C, st *state.State) []binarystorage.Metadata {
storage, err := st.ToolsStorage()
c.Assert(err, jc.ErrorIsNil)
defer storage.Close()
metadata, err := storage.AllMetadata()
c.Assert(err, jc.ErrorIsNil)
return metadata
func (s *caasToolsSuite) SetUpTest(c *gc.C) {
s.ControllerModelType = state.ModelTypeCAAS
s.baseToolsSuite.SetUpTest(c)
}

func (s *toolsSuite) testDownload(c *gc.C, tools *coretools.Tools, uuid string) []byte {
resp := s.downloadRequest(c, tools.Version, uuid)
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
func (s *caasToolsSuite) TestToolDownloadNotSharedCAASController(c *gc.C) {
closer, testStorage, _ := envtesting.CreateLocalTestStorage(c)
defer closer.Close()

const n = 8
states := []*state.State{}
for i := 0; i < n; i++ {
testState := s.Factory.MakeModel(c, nil)
defer testState.Close()
states = append(states, testState)
}

var mut sync.Mutex
resolutions := 0
envtools.RegisterToolsDataSourceFunc("local storage", func(environs.Environ) (simplestreams.DataSource, error) {
// Add some delay to make sure all goroutines are waiting.
time.Sleep(10 * time.Millisecond)
mut.Lock()
defer mut.Unlock()
resolutions++
return storage.NewStorageSimpleStreamsDataSource("test datasource", testStorage, "tools", simplestreams.CUSTOM_CLOUD_DATA, false), nil
})
defer envtools.UnregisterToolsDataSourceFunc("local storage")

tool := version.MustParseBinary("2.9.99-ubuntu-amd64")
stream := "released"
tools, err := envtesting.UploadFakeToolsVersions(testStorage, stream, stream, tool)
c.Assert(err, jc.ErrorIsNil)
c.Assert(data, gc.HasLen, int(tools.Size))
c.Assert(tools, gc.HasLen, 1)

hash := sha256.New()
hash.Write(data)
c.Assert(fmt.Sprintf("%x", hash.Sum(nil)), gc.Equals, tools.SHA256)
return data
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
i := i
go func() {
defer wg.Done()
s.testDownload(c, tools[0], states[i].ModelUUID())
}()
}
wg.Wait()

c.Assert(resolutions, gc.Equals, n)
}
1 change: 1 addition & 0 deletions provider/lxd/credentials.go
View file
Expand Up @@ -677,6 +677,7 @@ func configDirs() []string {
dirs = append(dirs, filepath.Join(utils.Home(), ".config", "lxc"))
if runtime.GOOS == "linux" {
dirs = append(dirs, filepath.Join(utils.Home(), "snap", "lxd", "current", ".config", "lxc"))
dirs = append(dirs, filepath.Join(utils.Home(), "snap", "lxd", "common", "config"))
}
return dirs
}
14 changes: 11 additions & 3 deletions provider/lxd/credentials_test.go
View file
Expand Up @@ -122,6 +122,7 @@ func (s *credentialsSuite) TestDetectCredentialsGeneratesCertFailsToWriteOnError
deps.certReadWriter.EXPECT().Read(path).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read(filepath.Join(utils.Home(), ".config", "lxc")).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/common/config").Return(nil, nil, os.ErrNotExist)
deps.certGenerator.EXPECT().Generate(true, true).Return(nil, nil, errors.Errorf("bad"))

_, err := deps.provider.DetectCredentials("")
Expand All @@ -138,6 +139,7 @@ func (s *credentialsSuite) TestDetectCredentialsGeneratesCertFailsToGetCertifica
deps.certReadWriter.EXPECT().Read(path).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read(filepath.Join(utils.Home(), ".config", "lxc")).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/common/config").Return(nil, nil, os.ErrNotExist)
deps.certGenerator.EXPECT().Generate(true, true).Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.certReadWriter.EXPECT().Write(path, []byte(coretesting.CACert), []byte(coretesting.CAKey)).Return(errors.Errorf("bad"))

Expand Down Expand Up @@ -171,6 +173,7 @@ func (s *credentialsSuite) TestRemoteDetectCredentials(c *gc.C) {
},
},
}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/common/config/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.configReader.EXPECT().ReadCert("snap/lxd/current/.config/lxc/servercerts/nuc1.crt").Return([]byte(coretesting.ServerCert), nil)
deps.server.EXPECT().GetCertificate(s.clientCertFingerprint(c)).Return(nil, "", nil)
Expand Down Expand Up @@ -216,7 +219,8 @@ func (s *credentialsSuite) TestRemoteDetectCredentialsNoRemoteCert(c *gc.C) {

deps.configReader.EXPECT().ReadConfig(path.Join(osenv.JujuXDGDataHomePath("lxd"), "config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), ".config/lxc/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/current/.config/lxc/config.yml")).Return(lxd.LXCConfig{
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/current/.config/lxc/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/common/config/config.yml")).Return(lxd.LXCConfig{
DefaultRemote: "localhost",
Remotes: map[string]lxd.LXCRemoteConfig{
"nuc1": {
Expand All @@ -227,8 +231,8 @@ func (s *credentialsSuite) TestRemoteDetectCredentialsNoRemoteCert(c *gc.C) {
},
},
}, nil)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.configReader.EXPECT().ReadCert("snap/lxd/current/.config/lxc/servercerts/nuc1.crt").Return(nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/common/config").Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.configReader.EXPECT().ReadCert("snap/lxd/common/config/servercerts/nuc1.crt").Return(nil, os.ErrNotExist)
deps.server.EXPECT().GetCertificate(s.clientCertFingerprint(c)).Return(nil, "", nil)
deps.server.EXPECT().ServerCertificate().Return(coretesting.ServerCert)

Expand Down Expand Up @@ -273,6 +277,7 @@ func (s *credentialsSuite) TestRemoteDetectCredentialsWithConfigFailure(c *gc.C)
deps.configReader.EXPECT().ReadConfig(path.Join(osenv.JujuXDGDataHomePath("lxd"), "config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), ".config/lxc/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/current/.config/lxc/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/common/config/config.yml")).Return(lxd.LXCConfig{}, nil)

deps.server.EXPECT().GetCertificate(s.clientCertFingerprint(c)).Return(nil, "", errors.New("bad"))

Expand Down Expand Up @@ -303,6 +308,7 @@ func (s *credentialsSuite) TestRemoteDetectCredentialsWithCertFailure(c *gc.C) {
},
},
}, nil)
deps.configReader.EXPECT().ReadConfig(path.Join(utils.Home(), "snap/lxd/common/config/config.yml")).Return(lxd.LXCConfig{}, nil)
deps.certReadWriter.EXPECT().Read(path.Join(utils.Home(), "snap/lxd/current/.config/lxc")).Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.configReader.EXPECT().ReadCert("snap/lxd/current/.config/lxc/servercerts/nuc1.crt").Return(nil, errors.New("bad"))
deps.server.EXPECT().GetCertificate(s.clientCertFingerprint(c)).Return(nil, "", errors.New("bad"))
Expand All @@ -324,6 +330,7 @@ func (s *credentialsSuite) TestRegisterCredentials(c *gc.C) {
deps.certReadWriter.EXPECT().Read(path).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read(filepath.Join(utils.Home(), ".config", "lxc")).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/common/config").Return(nil, nil, os.ErrNotExist)
deps.certGenerator.EXPECT().Generate(true, true).Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.certReadWriter.EXPECT().Write(path, []byte(coretesting.CACert), []byte(coretesting.CAKey)).Return(nil)

Expand Down Expand Up @@ -365,6 +372,7 @@ func (s *credentialsSuite) TestRegisterCredentialsWithAlternativeCloudName(c *gc
deps.certReadWriter.EXPECT().Read(path).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read(filepath.Join(utils.Home(), ".config", "lxc")).Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/current/.config/lxc").Return(nil, nil, os.ErrNotExist)
deps.certReadWriter.EXPECT().Read("snap/lxd/common/config").Return(nil, nil, os.ErrNotExist)
deps.certGenerator.EXPECT().Generate(true, true).Return([]byte(coretesting.CACert), []byte(coretesting.CAKey), nil)
deps.certReadWriter.EXPECT().Write(path, []byte(coretesting.CACert), []byte(coretesting.CAKey)).Return(nil)
deps.server.EXPECT().GetCertificate(s.clientCertFingerprint(c)).Return(nil, "", nil)
Expand Down

0 comments on commit 9983864

Please sign in to comment.