Merge pull request #15164 from manadart/2.9-into-3.0

#15164

Merge from 2.9 to bring forward:
- #15129 from jack-w-shaw/JUJU-2615_debug_log_for_caas
- #15141 from hmlanigan/198858701-refresh-switch-no-rev-change
- #15154 from manadart/2.9-migration-rollback
- #15157 from manadart/2.9-sync-agent-binary
- #15117 from cprivitere/whitelist-10-networks

Trivial conflicts.

api/common/logs.go

@@ -22,7 +22,7 @@ import (
 // closes the connection.
 type DebugLogParams struct {
 	// IncludeEntity lists entity tags to include in the response. Tags may
-	// finish with a '*' to match a prefix e.g.: unit-mysql-*, machine-2. If
+	// include '*' wildcards e.g.: unit-mysql-*, machine-2. If
 	// none are set, then all lines are considered included.
 	IncludeEntity []string
 	// IncludeModule lists logging modules to include in the response. If none
@@ -33,7 +33,7 @@ type DebugLogParams struct {
 	// are set all labels are considered included.
 	IncludeLabel []string
 	// ExcludeEntity lists entity tags to exclude from the response. As with
-	// IncludeEntity the values may finish with a '*'.
+	// IncludeEntity the values may include '*' wildcards.
 	ExcludeEntity []string
 	// ExcludeModule lists logging modules to exclude from the response. If a
 	// module is specified, all the submodules are also excluded.

cmd/juju/application/refresh.go

@@ -407,7 +407,8 @@ func (c *refreshCommand) Run(ctx *cmd.Context) error {
 		return errors.Trace(err)
 	}
 	charmID, err := factory.Run(cfg)
-	if err == nil {
+	switch {
+	case err == nil:
 		curl := charmID.URL
 		charmOrigin := charmID.Origin
 		if charmOrigin.Source == corecharm.CharmStore {
@@ -419,21 +420,22 @@ func (c *refreshCommand) Run(ctx *cmd.Context) error {
 			channel = fmt.Sprintf(" in channel %s", charmID.Origin.Channel.String())
 		}
 		ctx.Infof("Added %s charm %q, revision %d%s, to the model", charmOrigin.Source, curl.Name, curl.Revision, channel)
-	} else if errors.Is(err, refresher.ErrAlreadyUpToDate) {
-		if len(c.Resources) == 0 {
-			// Charm already up-to-date and no resources to refresh.
-			ctx.Infof(err.Error())
-			return nil
-		}
-	} else {
+	case errors.Is(err, refresher.ErrAlreadyUpToDate) && c.Channel.String() != oldOrigin.CoreCharmOrigin().Channel.String():
+		ctx.Infof("%s. Note: all future refreshes will now use channel %q", err.Error(), charmID.Origin.Channel.String())
+	case errors.Is(err, refresher.ErrAlreadyUpToDate) && len(c.Resources) == 0:
+		// Charm already up-to-date and no resources to refresh.
+		ctx.Infof(err.Error())
+		return nil
+	case errors.Is(err, refresher.ErrAlreadyUpToDate) && len(c.Resources) > 0:
+		ctx.Infof("%s. Attempt to update resources requested.", err.Error())
+	default:
 		if termErr, ok := errors.Cause(err).(*common.TermsRequiredError); ok {
 			return errors.Trace(termErr.UserErr())
 		}
 		return block.ProcessBlockedError(err, block.BlockChange)
 	}
 
 	// Next, upgrade resources.
-
 	resourceLister, err := c.NewResourceLister(apiRoot)
 	if err != nil {
 		return errors.Trace(err)

cmd/juju/application/refresh_test.go

@@ -149,6 +149,9 @@ func (s *BaseRefreshSuite) SetUpTest(c *gc.C) {
 		bindings: map[string]string{
 			"": network.AlphaSpaceName,
 		},
+		charmOrigin: commoncharm.Origin{
+			Risk: "stable",
+		},
 	}
 	s.modelConfigGetter = newMockModelConfigGetter()
 	s.resourceLister = mockResourceLister{}
@@ -614,6 +617,29 @@ func (s *RefreshSuite) TestUpgradeWithChannel(c *gc.C) {
 	})
 }
 
+func (s *RefreshSuite) TestUpgradeWithChannelNoNewCharmURL(c *gc.C) {
+	// Test setting a new charm channel, without an actual
+	// charm upgrade needed.
+	s.resolvedCharmURL = charm.MustParseURL("cs:quantal/foo-1")
+	s.resolvedChannel = csclientparams.BetaChannel
+	_, err := s.runRefresh(c, "foo", "--channel=beta")
+	c.Assert(err, jc.ErrorIsNil)
+
+	s.charmAPIClient.CheckCallNames(c, "GetCharmURLOrigin", "Get", "SetCharm")
+	s.charmAPIClient.CheckCall(c, 2, "SetCharm", model.GenerationMaster, application.SetCharmConfig{
+		ApplicationName: "foo",
+		CharmID: application.CharmID{
+			URL: s.resolvedCharmURL,
+			Origin: commoncharm.Origin{
+				Source:       "charm-store",
+				Architecture: arch.DefaultArchitecture,
+				Risk:         "beta",
+			},
+		},
+		EndpointBindings: map[string]string{},
+	})
+}
+
 func (s *RefreshSuite) TestRefreshShouldRespectDeployedChannelByDefault(c *gc.C) {
 	s.resolvedChannel = csclientparams.BetaChannel
 	_, err := s.runRefresh(c, "foo")

cmd/juju/commands/debuglog.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/juju/ansiterm"
 	"github.com/juju/cmd/v3"
+	"github.com/juju/collections/transform"
 	"github.com/juju/errors"
 	"github.com/juju/gnuflag"
 	"github.com/juju/juju/jujuclient"
@@ -46,7 +47,8 @@ machines and units can be seen in the output of `[1:] + "`juju status`" + `.
 
 The '--include' and '--exclude' options filter by entity. The entity can be
 a machine, unit, or application for vm models, but can be application only
-for k8s models.
+for k8s models. These filters support wildcards ` + "`*`" + ` if filtering on the
+entity full name (prefixed by ` + "`<entity type>-`" + `)
 
 The '--include-module' and '--exclude-module' options filter by (dotted)
 logging module name. The module name can be truncated such that all loggers
@@ -204,45 +206,50 @@ func (c *debugLogCommand) Init(args []string) error {
 		return errors.Trace(err)
 	}
 	isCaas := modelType == model.CAAS
-	c.params.IncludeEntity = c.processEntities(isCaas, c.params.IncludeEntity)
-	c.params.ExcludeEntity = c.processEntities(isCaas, c.params.ExcludeEntity)
+	if isCaas {
+		c.params.IncludeEntity = transform.Slice(c.params.IncludeEntity, c.parseCAASEntity)
+		c.params.ExcludeEntity = transform.Slice(c.params.ExcludeEntity, c.parseCAASEntity)
+	} else {
+		c.params.IncludeEntity = transform.Slice(c.params.IncludeEntity, c.parseEntity)
+		c.params.ExcludeEntity = transform.Slice(c.params.ExcludeEntity, c.parseEntity)
+	}
 	return cmd.CheckEmpty(args)
 }
 
-func (c *debugLogCommand) processEntities(isCAAS bool, entities []string) []string {
-	if entities == nil {
-		return nil
+func (c *debugLogCommand) parseEntity(entity string) string {
+	tag, err := names.ParseTag(entity)
+	switch {
+	case strings.Contains(entity, "*"):
+		return entity
+	case err == nil && (tag.Kind() == names.ApplicationTagKind || tag.Kind() == names.MachineTagKind || tag.Kind() == names.UnitTagKind):
+		return tag.String()
+	case names.IsValidMachine(entity):
+		return names.NewMachineTag(entity).String()
+	case names.IsValidUnit(entity):
+		return names.NewUnitTag(entity).String()
+	case names.IsValidApplication(entity):
+		// If the user asks for --include nova-compute, we should give all
+		// nova-compute units for IAAS models.
+		return names.UnitTagKind + "-" + entity + "-*"
+	default:
+		logger.Warningf("%q was not recognised as a valid application, machine or unit name", entity)
+		return entity
 	}
-	result := make([]string, len(entities))
-	for i, entity := range entities {
-		// A stringified unit or machine tag never match their "IsValid"
-		// function from names, so if the string value passed in is a valid
-		// machine or unit, then convert here.
-		if names.IsValidMachine(entity) {
-			entity = names.NewMachineTag(entity).String()
-		} else if names.IsValidUnit(entity) {
-			entity = names.NewUnitTag(entity).String()
-		} else {
-			// Now we want to deal with a special case. Both stringified
-			// machine tags and stringified units are valid application names.
-			// So here we use special knowledge about how tags are serialized to
-			// be able to give a better user experience.  If the user asks for
-			// --include nova-compute, we should give all nova-compute units.
-			if strings.HasPrefix(entity, names.UnitTagKind+"-") ||
-				strings.HasPrefix(entity, names.MachineTagKind+"-") {
-				// no-op pass through
-			} else if names.IsValidApplication(entity) {
-				// Assume that the entity refers to an application.
-				if isCAAS {
-					entity = names.NewApplicationTag(entity).String()
-				} else {
-					entity = names.UnitTagKind + "-" + entity + "-*"
-				}
-			}
-		}
-		result[i] = entity
+}
+
+func (c *debugLogCommand) parseCAASEntity(entity string) string {
+	tag, err := names.ParseTag(entity)
+	switch {
+	case strings.Contains(entity, "*"):
+		return entity
+	case err == nil && tag.Kind() == names.ApplicationTagKind:
+		return tag.String()
+	case names.IsValidApplication(entity):
+		return names.NewApplicationTag(entity).String()
+	default:
+		logger.Warningf("%q was not recognised as a valid application name. Only applications produce logs for CAAS models application", entity)
+		return entity
 	}
-	return result
 }
 
 type DebugLogAPI interface {

cmd/juju/commands/synctools.go

@@ -57,8 +57,8 @@ The online store will, of course, need to be contacted at some point to get
 the software.
 
 Examples:
-    juju sync-agent-binary --debug --version 2.0
-    juju sync-agent-binary --debug --version 2.0 --local-dir=/home/ubuntu/sync-agent-binary
+    juju sync-agent-binary --debug --agent-version 2.0
+    juju sync-agent-binary --debug --agent-version 2.0 --local-dir=/home/ubuntu/sync-agent-binary
 
 See also:
     upgrade-controller
@@ -75,7 +75,7 @@ func (c *syncAgentBinaryCommand) Info() *cmd.Info {
 
 func (c *syncAgentBinaryCommand) SetFlags(f *gnuflag.FlagSet) {
 	c.ModelCommandBase.SetFlags(f)
-	f.StringVar(&c.versionStr, "version", "", "Copy a specific major[.minor] version")
+	f.StringVar(&c.versionStr, "agent-version", "", "Copy a specific major[.minor] version")
 	f.BoolVar(&c.dryRun, "dry-run", false, "Don't copy, just print what would be copied")
 	f.BoolVar(&c.public, "public", false, "Tools are for a public cloud, so generate mirrors information")
 	f.StringVar(&c.source, "source", "", "Local source directory")
@@ -85,7 +85,7 @@ func (c *syncAgentBinaryCommand) SetFlags(f *gnuflag.FlagSet) {
 
 func (c *syncAgentBinaryCommand) Init(args []string) error {
 	if c.versionStr == "" {
-		return errors.NewNotValid(nil, "--version is required")
+		return errors.NewNotValid(nil, "--agent-version is required")
 	}
 	var err error
 	if c.targetVersion, err = version.Parse(c.versionStr); err != nil {

cmd/juju/commands/synctools_test.go

@@ -73,21 +73,21 @@ type syncToolCommandTestCase struct {
 var syncToolCommandTests = []syncToolCommandTestCase{
 	{
 		description: "minimum argument",
-		args:        []string{"--version", "2.9.99", "-m", "test-target"},
+		args:        []string{"--agent-version", "2.9.99", "-m", "test-target"},
 	},
 	{
 		description: "specifying also the synchronization source",
-		args:        []string{"--version", "2.9.99", "-m", "test-target", "--source", "/foo/bar"},
+		args:        []string{"--agent-version", "2.9.99", "-m", "test-target", "--source", "/foo/bar"},
 		source:      "/foo/bar",
 	},
 	{
 		description: "just make a dry run",
-		args:        []string{"--version", "2.9.99", "-m", "test-target", "--dry-run"},
+		args:        []string{"--agent-version", "2.9.99", "-m", "test-target", "--dry-run"},
 		dryRun:      true,
 	},
 	{
 		description: "specified public (ignored by API)",
-		args:        []string{"--version", "2.9.99", "-m", "test-target", "--public"},
+		args:        []string{"--agent-version", "2.9.99", "-m", "test-target", "--public"},
 		public:      false,
 	},
 }
@@ -129,7 +129,8 @@ func (s *syncToolSuite) TestSyncToolsCommand(c *gc.C) {
 
 func (s *syncToolSuite) TestSyncToolsCommandTargetDirectory(c *gc.C) {
 	dir := c.MkDir()
-	ctrl, run := s.getSyncAgentBinariesCommand(c, "--version", "2.9.99", "-m", "test-target", "--local-dir", dir, "--stream", "proposed")
+	ctrl, run := s.getSyncAgentBinariesCommand(
+		c, "--agent-version", "2.9.99", "-m", "test-target", "--local-dir", dir, "--stream", "proposed")
 	defer ctrl.Finish()
 
 	called := false
@@ -155,7 +156,8 @@ func (s *syncToolSuite) TestSyncToolsCommandTargetDirectory(c *gc.C) {
 
 func (s *syncToolSuite) TestSyncToolsCommandTargetDirectoryPublic(c *gc.C) {
 	dir := c.MkDir()
-	ctrl, run := s.getSyncAgentBinariesCommand(c, "--version", "2.9.99", "-m", "test-target", "--local-dir", dir, "--public")
+	ctrl, run := s.getSyncAgentBinariesCommand(
+		c, "--agent-version", "2.9.99", "-m", "test-target", "--local-dir", dir, "--public")
 	defer ctrl.Finish()
 
 	called := false
@@ -187,7 +189,8 @@ func (s *syncToolSuite) TestAPIAdapterUploadTools(c *gc.C) {
 }
 
 func (s *syncToolSuite) TestAPIAdapterBlockUploadTools(c *gc.C) {
-	ctrl, run := s.getSyncAgentBinariesCommand(c, "-m", "test-target", "--version", "2.9.99", "--local-dir", c.MkDir(), "--stream", "released")
+	ctrl, run := s.getSyncAgentBinariesCommand(
+		c, "-m", "test-target", "--agent-version", "2.9.99", "--local-dir", c.MkDir(), "--stream", "released")
 	defer ctrl.Finish()
 
 	syncTools = func(sctx *sync.SyncContext) error {

provider/equinix/environ.go

@@ -302,6 +302,7 @@ func getCloudConfig(args environs.StartInstanceParams) (cloudinit.CloudConfig, e
 			}
 		}
 	}
+	iptablesDefault = append(iptablesDefault, "iptables -A INPUT -s 10.0.0.0/8 -j ACCEPT")
 	iptablesDefault = append(iptablesDefault, "iptables -A INPUT -j DROP")
 
 	cloudCfg.AddScripts(

state/application.go

@@ -1670,15 +1670,24 @@ func (a *Application) SetCharm(cfg SetCharmConfig) (err error) {
 		}}
 
 		if *a.doc.CharmURL == cfg.Charm.URL().String() {
+			updates := bson.D{
+				{"cs-channel", channel},
+				{"forcecharm", cfg.ForceUnits},
+			}
+			// Local charms will not have a channel in their charm origin
+			// TODO: (hml) 2023-02-03
+			// With juju 3.0, SetCharm should always have a CharmOrigin.
+			// Compatibility with the Update application facade method
+			// is no longer necessary.
+			if cfg.CharmOrigin != nil && cfg.CharmOrigin.Channel != nil {
+				updates = append(updates, bson.DocElem{"charm-origin.channel", cfg.CharmOrigin.Channel})
+			}
 			// Charm URL already set; just update the force flag and channel.
 			ops = append(ops, txn.Op{
 				C:      applicationsC,
 				Id:     a.doc.DocID,
 				Assert: txn.DocExists,
-				Update: bson.D{{"$set", bson.D{
-					{"cs-channel", channel},
-					{"forcecharm", cfg.ForceUnits},
-				}}},
+				Update: bson.D{{"$set", updates}},
 			})
 		} else {
 			// Check if the new charm specifies a relation max limit
@@ -1709,6 +1718,10 @@ func (a *Application) SetCharm(cfg SetCharmConfig) (err error) {
 			newCharmModifiedVersion++
 		}
 
+		// TODO: (hml) 2023-02-03
+		// With juju 3.0, SetCharm should always have a CharmOrigin.
+		// Compatibility with the Update application facade method
+		// is no longer necessary. Modify checks appropriately.
 		if cfg.CharmOrigin != nil {
 			origin := a.doc.CharmOrigin
 			// If either the charm origin ID or Hash is set before a charm is
@@ -1778,7 +1791,6 @@ func (a *Application) SetCharm(cfg SetCharmConfig) (err error) {
 			// ErrNoOperations on the other hand means there's nothing to update.
 			return nil, errors.Trace(err)
 		}
-
 		return ops, nil
 	}
 

state/application_test.go

@@ -128,6 +128,26 @@ func (s *ApplicationSuite) TestSetCharmCharmOrigin(c *gc.C) {
 	c.Assert(obtainedOrigin, gc.DeepEquals, origin)
 }
 
+func (s *ApplicationSuite) TestSetCharmUpdateChannelURLNoChange(c *gc.C) {
+	sch := s.AddMetaCharm(c, "mysql", metaBase, 2)
+
+	origin := s.mysql.CharmOrigin()
+	origin.Channel = &state.Channel{Risk: "stable"}
+
+	cfg := state.SetCharmConfig{
+		Charm:       sch,
+		CharmOrigin: origin,
+	}
+	err := s.mysql.SetCharm(cfg)
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(s.mysql.CharmOrigin().Channel.Risk, gc.DeepEquals, "stable")
+
+	cfg.CharmOrigin.Channel.Risk = "candidate"
+	err = s.mysql.SetCharm(cfg)
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(s.mysql.CharmOrigin().Channel.Risk, gc.DeepEquals, "candidate")
+}
+
 func (s *ApplicationSuite) TestSetCharmCharmOriginNoChange(c *gc.C) {
 	// Add a compatible charm.
 	sch := s.AddMetaCharm(c, "mysql", metaBase, 2)

state/applicationoffers.go

@@ -143,7 +143,7 @@ func (s *applicationOffers) AllApplicationOffers() (offers []*crossmodel.Applica
 	var docs []applicationOfferDoc
 	err := applicationOffersCollection.Find(bson.D{}).All(&docs)
 	if err != nil {
-		return nil, errors.Errorf("cannot get all application offers")
+		return nil, errors.Annotate(err, "getting application offer documents")
 	}
 	for _, doc := range docs {
 		offer, err := s.makeApplicationOffer(doc)

state/logs.go

@@ -762,8 +762,10 @@ func makeEntityPattern(entities []string) string {
 	var patterns []string
 	for _, entity := range entities {
 		// Convert * wildcard to the regex equivalent. This is safe
-		// because * never appears in entity names.
-		patterns = append(patterns, strings.Replace(entity, "*", ".*", -1))
+		// because * never appears in entity names. Escape any other regex.
+		escaped := regexp.QuoteMeta(entity)
+		unescapedWildcards := strings.Replace(escaped, regexp.QuoteMeta("*"), ".*", -1)
+		patterns = append(patterns, unescapedWildcards)
 	}
 	return `^(` + strings.Join(patterns, "|") + `)$`
 }