WIP: stream fallback

apiserver/facades/agent/provisioner/provisioninginfo.go

@@ -403,8 +403,8 @@ func (p *ProvisionerAPI) availableImageMetadata(m *state.Machine, env environs.E
 // constructImageConstraint returns model-specific criteria used to look for image metadata.
 func (p *ProvisionerAPI) constructImageConstraint(m *state.Machine, env environs.Environ) (*imagemetadata.ImageConstraint, error) {
 	lookup := simplestreams.LookupParams{
-		Series: []string{m.Series()},
-		Stream: env.Config().ImageStream(),
+		Series:  []string{m.Series()},
+		Streams: []string{env.Config().ImageStream()},
 	}
 
 	mcons, err := m.Constraints()
@@ -466,11 +466,15 @@ func (p *ProvisionerAPI) findImageMetadata(imageConstraint *imagemetadata.ImageC
 // imageMetadataFromState returns image metadata stored in state
 // that matches given criteria.
 func (p *ProvisionerAPI) imageMetadataFromState(constraint *imagemetadata.ImageConstraint) ([]params.CloudImageMetadata, error) {
+	if len(constraint.Streams) != 1 {
+		return nil, errors.Errorf("expected one stream in image constraint, got %q",
+			strings.Join(constraint.Streams, ","))
+	}
 	filter := cloudimagemetadata.MetadataFilter{
 		Series: constraint.Series,
 		Arches: constraint.Arches,
 		Region: constraint.Region,
-		Stream: constraint.Stream,
+		Stream: constraint.Streams[0],
 	}
 	stored, err := p.st.CloudImageMetadataStorage.FindMetadata(filter)
 	if err != nil {
@@ -528,8 +532,8 @@ func (p *ProvisionerAPI) imageMetadataFromDataSources(env environs.Environ, cons
 		// TODO (anastasiamac 2016-08-24) This is a band-aid solution.
 		// Once correct value is read from simplestreams, this needs to go.
 		// Bug# 1616295
-		if result.Stream == "" {
-			result.Stream = constraint.Stream
+		if result.Stream == "" && len(constraint.Streams) > 0 {
+			result.Stream = constraint.Streams[0]
 		}
 		if result.Stream == "" {
 			result.Stream = cfg.ImageStream()

environs/bootstrap/bootstrap.go

@@ -585,7 +585,7 @@ func bootstrapImageMetadata(
 	// This constraint will search image metadata for all supported architectures and series.
 	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
 		CloudSpec: region,
-		Stream:    environ.Config().ImageStream(),
+		Streams:   []string{environ.Config().ImageStream()},
 	})
 	logger.Debugf("constraints for image metadata lookup %v", imageConstraint)
 

environs/gui/export_test.go

@@ -14,7 +14,7 @@ var (
 
 func NewConstraint(stream string, majorVersion int) *constraint {
 	return &constraint{
-		LookupParams: simplestreams.LookupParams{Stream: stream},
+		LookupParams: simplestreams.LookupParams{Streams: []string{stream}},
 		majorVersion: majorVersion,
 	}
 }

environs/gui/simplestreams.go

@@ -53,7 +53,7 @@ func FetchMetadata(stream string, sources ...simplestreams.DataSource) ([]*Metad
 	params := simplestreams.GetMetadataParams{
 		StreamsVersion: streamsVersion,
 		LookupConstraint: &constraint{
-			LookupParams: simplestreams.LookupParams{Stream: stream},
+			LookupParams: simplestreams.LookupParams{Streams: []string{stream}},
 			majorVersion: jujuversion.Current.Major,
 		},
 		ValueParams: simplestreams.ValueParams{
@@ -63,12 +63,14 @@ func FetchMetadata(stream string, sources ...simplestreams.DataSource) ([]*Metad
 			ValueTemplate:   Metadata{},
 		},
 	}
-	items, _, err := simplestreams.GetMetadata(sources, params)
+	results, err := simplestreams.GetMetadata(sources, params)
 	if err != nil {
 		return nil, errors.Annotate(err, "error fetching simplestreams metadata")
 	}
-	allMeta := make([]*Metadata, len(items))
-	for i, item := range items {
+	// GetMetadata returns NotFound if there are 0 results.
+	result := results[0]
+	allMeta := make([]*Metadata, len(result.Items))
+	for i, item := range result.Items {
 		allMeta[i] = item.(*Metadata)
 	}
 	sort.Sort(byVersion(allMeta))
@@ -112,7 +114,11 @@ type constraint struct {
 // IndexIds generates a string array representing index ids formed similarly to
 // an ISCSI qualified name (IQN).
 func (c *constraint) IndexIds() []string {
-	return []string{contentId(c.Stream)}
+	var results []string
+	for _, stream := range c.Streams {
+		results = append(results, contentId(stream))
+	}
+	return results
 }
 
 // ProductIds generates a string array representing product ids formed

environs/gui/simplestreams_test.go

@@ -218,7 +218,7 @@ func (s *simplestreamsSuite) TestConstraint(c *gc.C) {
 
 	c.Assert(constraint.Endpoint, gc.Equals, "")
 	c.Assert(constraint.Region, gc.Equals, "")
-	c.Assert(constraint.Stream, gc.Equals, "test-stream")
+	c.Assert(constraint.Streams, gc.DeepEquals, []string{"test-stream"})
 }
 
 var guiData = map[string]string{

environs/imagedownloads/simplestreams.go

@@ -99,18 +99,19 @@ func Fetch(
 			ValueTemplate: Metadata{},
 		},
 	}
-	items, resolveInfo, err := simplestreams.GetMetadata(src, params)
+	results, err := simplestreams.GetMetadata(src, params)
 	if err != nil {
-		return nil, resolveInfo, err
+		return nil, nil, err
 	}
-	md := make([]*Metadata, len(items))
-	for i, im := range items {
+	// GetMetadata returns NotFound if there are no results.
+	md := make([]*Metadata, len(results[0].Items))
+	for i, im := range results[0].Items {
 		md[i] = im.(*Metadata)
 	}
 
 	Sort(md)
 
-	return md, resolveInfo, nil
+	return md, results[0].ResolveInfo, nil
 }
 
 func validateArgs(arch, release, ftype string) error {

environs/imagemetadata/simplestreams.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"sort"
 
+	"github.com/juju/errors"
 	"github.com/juju/utils"
 	"github.com/juju/utils/arch"
 	"github.com/juju/utils/series"
@@ -189,17 +190,32 @@ func (ic *ImageConstraint) IndexIds() []string {
 
 // ProductIds generates a string array representing product ids formed similarly to an ISCSI qualified name (IQN).
 func (ic *ImageConstraint) ProductIds() ([]string, error) {
-	stream := idStream(ic.Stream)
+	streams := ic.Streams
+	if len(streams) == 0 {
+		// If there are no streams specified use the default ""
+		streams = []string{""}
+	}
 	nrArches := len(ic.Arches)
 	nrSeries := len(ic.Series)
-	ids := make([]string, nrArches*nrSeries)
-	for i, arch := range ic.Arches {
-		for j, ser := range ic.Series {
-			version, err := series.SeriesVersion(ser)
-			if err != nil {
-				return nil, err
+	nrStreams := len(streams)
+	ids := make([]string, 0, nrArches*nrSeries*nrStreams)
+
+	addIdsForStream := func(stream string) error {
+		for _, arch := range ic.Arches {
+			for _, ser := range ic.Series {
+				version, err := series.SeriesVersion(ser)
+				if err != nil {
+					return errors.Trace(err)
+				}
+				ids = append(ids, fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch))
 			}
-			ids[j*nrArches+i] = fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch)
+		}
+		return nil
+	}
+
+	for _, stream := range streams {
+		if err := addIdsForStream(idStream(stream)); err != nil {
+			return nil, errors.Trace(err)
 		}
 	}
 	return ids, nil
@@ -244,18 +260,19 @@ func Fetch(
 			ValueTemplate: ImageMetadata{},
 		},
 	}
-	items, resolveInfo, err := simplestreams.GetMetadata(sources, params)
+	results, err := simplestreams.GetMetadata(sources, params)
 	if err != nil {
-		return nil, resolveInfo, err
+		return nil, nil, err
 	}
-	metadata := make([]*ImageMetadata, len(items))
-	for i, md := range items {
+	// GetMetadata returns NotFound if there are 0 results.
+	metadata := make([]*ImageMetadata, len(results[0].Items))
+	for i, md := range results[0].Items {
 		metadata[i] = md.(*ImageMetadata)
 	}
 	// Sorting the metadata is not strictly necessary, but it ensures consistent ordering for
 	// all compilers, and it just makes it easier to look at the data.
 	Sort(metadata)
-	return metadata, resolveInfo, nil
+	return metadata, results[0].ResolveInfo, nil
 }
 
 // Sort sorts a slice of ImageMetadata in ascending order of their id

environs/imagemetadata/simplestreams_test.go

@@ -319,7 +319,7 @@ func (s *productSpecSuite) TestIdWithDefaultStream(c *gc.C) {
 		Arches: []string{"amd64"},
 	})
 	for _, stream := range []string{"", "released"} {
-		imageConstraint.Stream = stream
+		imageConstraint.Streams = []string{stream}
 		ids, err := imageConstraint.ProductIds()
 		c.Assert(err, jc.ErrorIsNil)
 		c.Assert(ids, gc.DeepEquals, []string{"com.ubuntu.cloud:server:12.04:amd64"})
@@ -328,9 +328,9 @@ func (s *productSpecSuite) TestIdWithDefaultStream(c *gc.C) {
 
 func (s *productSpecSuite) TestId(c *gc.C) {
 	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
-		Series: []string{"precise"},
-		Arches: []string{"amd64"},
-		Stream: "daily",
+		Series:  []string{"precise"},
+		Arches:  []string{"amd64"},
+		Streams: []string{"daily"},
 	})
 	ids, err := imageConstraint.ProductIds()
 	c.Assert(err, jc.ErrorIsNil)
@@ -339,9 +339,9 @@ func (s *productSpecSuite) TestId(c *gc.C) {
 
 func (s *productSpecSuite) TestIdMultiArch(c *gc.C) {
 	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
-		Series: []string{"precise"},
-		Arches: []string{"amd64", "i386"},
-		Stream: "daily",
+		Series:  []string{"precise"},
+		Arches:  []string{"amd64", "i386"},
+		Streams: []string{"daily"},
 	})
 	ids, err := imageConstraint.ProductIds()
 	c.Assert(err, jc.ErrorIsNil)

environs/imagemetadata/validation.go

@@ -29,9 +29,9 @@ func ValidateImageMetadata(params *simplestreams.MetadataLookupParams) ([]string
 			Region:   params.Region,
 			Endpoint: params.Endpoint,
 		},
-		Series: []string{params.Series},
-		Arches: params.Architectures,
-		Stream: params.Stream,
+		Series:  []string{params.Series},
+		Arches:  params.Architectures,
+		Streams: []string{params.Stream},
 	})
 	matchingImages, resolveInfo, err := Fetch(params.Sources, imageConstraint)
 	if err != nil {

environs/simplestreams/simplestreams.go

@@ -96,10 +96,10 @@ type LookupParams struct {
 	CloudSpec
 	Series []string
 	Arches []string
-	// Stream can be "" or "released" for the default "released" stream,
+	// Streams can be "" or "released" for the default "released" stream,
 	// or "daily" for daily images, or any other stream that the available
 	// simplestreams metadata supports.
-	Stream string
+	Streams []string
 }
 
 func (p LookupParams) Params() LookupParams {
@@ -384,12 +384,28 @@ type GetMetadataParams struct {
 	StreamsVersion   string
 	LookupConstraint LookupConstraint
 	ValueParams      ValueParams
-}
-
-// GetMetadata returns metadata records matching the specified constraint,looking in each source for signed metadata.
-// If onlySigned is false and no signed metadata is found in a source, the source is used to look for unsigned metadata.
-// Each source is tried in turn until at least one signed (or unsigned) match is found.
-func GetMetadata(sources []DataSource, params GetMetadataParams) (items []interface{}, resolveInfo *ResolveInfo, err error) {
+	UseAllSources    bool
+}
+
+type MetadataResult struct {
+	Items       []interface{}
+	ResolveInfo *ResolveInfo
+}
+
+// GetMetadata returns metadata records matching the specified
+// constraint,looking in each source for signed metadata. If
+// onlySigned is false and no signed metadata is found in a source,
+// the source is used to look for unsigned metadata. Each source is
+// tried in turn - if AllResults is false then the results from the
+// first signed (or unsigned) datasource containing an index are
+// returned. If UseAllSources is true results are returned for all
+// sources with an index.
+func GetMetadata(sources []DataSource, params GetMetadataParams) ([]MetadataResult, error) {
+	var (
+		items       []interface{}
+		resolveInfo *ResolveInfo
+		err         error
+	)
 	for _, source := range sources {
 		logger.Tracef("searching for signed metadata in datasource %q", source.Description())
 		items, resolveInfo, err = getMaybeSignedMetadata(source, params, true)
@@ -406,7 +422,7 @@ func GetMetadata(sources []DataSource, params GetMetadataParams) (items []interf
 		// no matching products is an internal error only
 		err = nil
 	}
-	return items, resolveInfo, err
+	return []MetadataResult{{Items: items, ResolveInfo: resolveInfo}}, err
 }
 
 // getMaybeSignedMetadata returns metadata records matching the specified constraint in params.

environs/simplestreams/simplestreams_test.go

@@ -350,10 +350,11 @@ func (s *simplestreamsSuite) TestGetMetadataNoMatching(c *gc.C) {
 		ValueParams:      simplestreams.ValueParams{DataType: "image-ids"},
 	}
 
-	items, resolveInfo, err := simplestreams.GetMetadata(sources, params)
+	results, err := simplestreams.GetMetadata(sources, params)
 	c.Assert(err, jc.ErrorIsNil)
-	c.Assert(items, gc.HasLen, 0)
-	c.Assert(resolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
+	c.Assert(results, gc.HasLen, 1)
+	c.Assert(results[0].Items, gc.HasLen, 0)
+	c.Assert(results[0].ResolveInfo, gc.DeepEquals, &simplestreams.ResolveInfo{
 		Source:    "test",
 		Signed:    false,
 		IndexURL:  "test:/daily/streams/v1/index.json",

environs/simplestreams/testing/testing.go

@@ -790,7 +790,7 @@ func (s *LocalLiveSimplestreamsSuite) TestGetProductsPathInvalidProductSpec(c *g
 		CloudSpec: s.ValidConstraint.Params().CloudSpec,
 		Series:    []string{"precise"},
 		Arches:    []string{"bad"},
-		Stream:    "spec",
+		Streams:   []string{"spec"},
 	})
 	_, err = indexRef.GetProductsPath(ic)
 	c.Assert(err, gc.NotNil)