Merge pull request #16462 from hpidcock/merge-3.3-main-20231019

#16462

Forward ports:
- #16426
- #16435
- #16445
- #16452
- #16451
- #16453
- #16455
- #16456
- #16457
- #16460
- #16461

Conflicts:
- apiserver/facades/agent/provisioner/imagemetadata_test.go
- environs/bootstrap/bootstrap.go
- environs/simplestreams/datasource.go
- environs/simplestreams/datasource_test.go
- go.sum
- worker/dbaccessor/package_test.go
- worker/firewaller/firewaller_test.go
- worker/modelcache/worker_test.go
- worker/uniter/charm/bundles_test.go
- worker/uniter/uniter_test.go
- worker/uniter/util_test.go
- cmd/juju/status/status_internal_test.go

CONTRIBUTING.md

@@ -50,12 +50,12 @@ that Go versions are not incremented during a release cycle. This means that
 branch may lag by one version or so.  Check the `go.mod` file at the root of
 the project for the targeted version of Go, as this is authoritative.
 
-For example, the following indicates that Go 1.20 is targeted:
+For example, the following indicates that Go 1.21 is targeted:
 
 ```
 module github.com/juju/juju
 
-go 1.20
+go 1.21
 ```
 
 ### Official distribution
@@ -66,7 +66,7 @@ Go can be [installed](https://golang.org/doc/install#install) from the official
 
 [Snap](https://snapcraft.io/go) may also be used to install Go on Linux.
 
-    snap install go --channel=1.19/stable --classic
+    snap install go --channel=1.21/stable --classic
 
 ## Build Juju and its dependencies
 

Makefile

@@ -181,10 +181,10 @@ ifeq ($(VERBOSE_CHECK), 1)
 endif
 
 define link_flags_version
-	-X $(PROJECT)/version.GitCommit=$(GIT_COMMIT) \
-	-X $(PROJECT)/version.GitTreeState=$(GIT_TREE_STATE) \
-	-X $(PROJECT)/version.build=$(JUJU_BUILD_NUMBER) \
-	-X $(PROJECT)/version.GoBuildTags=$(FINAL_BUILD_TAGS)
+-X $(PROJECT)/version.GitCommit=$(GIT_COMMIT) \
+-X $(PROJECT)/version.GitTreeState=$(GIT_TREE_STATE) \
+-X $(PROJECT)/version.build=$(JUJU_BUILD_NUMBER) \
+-X $(PROJECT)/version.GoBuildTags=$(FINAL_BUILD_TAGS)
 endef
 
 # Compile with debug flags if requested.
@@ -421,7 +421,7 @@ race-test:
 ## race-test: Verify Juju code using unit tests with the race detector enabled
 	+make run-tests TEST_ARGS="$(TEST_ARGS) -race"
 
-.PHONY: run-tests run-go-tests
+.PHONY: run-tests run-go-tests go-test-alias
 # Can't make the length of the TMP dir too long or it hits socket name length issues.
 run-tests: musl-install-if-missing dqlite-install-if-missing
 ## run-tests: Run the unit tests
@@ -459,6 +459,18 @@ run-go-tests: musl-install-if-missing dqlite-install-if-missing
 		CGO_ENABLED=1 \
 		go test -v -mod=$(JUJU_GOMOD_MODE) -tags=$(FINAL_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -ldflags ${CGO_LINK_FLAGS} -test.timeout=$(TEST_TIMEOUT) ${TEST_PACKAGES} -check.v -check.f $(TEST_FILTER)
 
+go-test-alias: musl-install-if-missing dqlite-install-if-missing
+## go-test-alias: Prints out an alias command for easy running of tests.
+	$(eval PPATH := "PATH")
+	@echo alias jt=\'PATH=\"${MUSL_BIN_PATH}:$$${PPATH}\" \
+		CC=\"musl-gcc\" \
+		CGO_CFLAGS=\"-I${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}/include\" \
+		CGO_LDFLAGS=\"-L${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH} -luv -lraft -ldqlite -llz4 -lsqlite3\" \
+		CGO_LDFLAGS_ALLOW=\""(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"\" \
+		LD_LIBRARY_PATH=\"${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}\" \
+		CGO_ENABLED=\"1\" \
+		go test -mod=\"$(JUJU_GOMOD_MODE)\" -tags=\"$(FINAL_BUILD_TAGS)\" -ldflags \"${CGO_LINK_FLAGS}\"\'
+
 .PHONY: install
 install: rebuild-schema go-install
 ## install: Install Juju binaries with a rebuilt schema

apiserver/facades/agent/provisioner/imagemetadata_test.go

@@ -5,6 +5,8 @@ package provisioner_test
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 
 	jc "github.com/juju/testing/checkers"
 	gc "gopkg.in/check.v1"
@@ -37,9 +39,16 @@ var _ = gc.Suite(&ImageMetadataSuite{})
 func (s *ImageMetadataSuite) SetUpSuite(c *gc.C) {
 	s.provisionerSuite.SetUpSuite(c)
 
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	}))
+	s.AddCleanup(func(c *gc.C) {
+		server.Close()
+	})
+
 	// Make sure that there is nothing in data sources.
 	// Each individual tests will decide if it needs metadata there.
-	imagetesting.PatchOfficialDataSources(&s.CleanupSuite, "test:/daily")
+	imagetesting.PatchOfficialDataSources(&s.CleanupSuite, server.URL)
 	s.PatchValue(&imagemetadata.SimplestreamsImagesPublicKey, sstesting.SignedMetadataPublicKey)
 	s.PatchValue(&keys.JujuPublicKey, sstesting.SignedMetadataPublicKey)
 	useTestImageData(c, nil)
@@ -156,6 +165,7 @@ func (s *ImageMetadataSuite) assertImageMetadataResults(
 ) {
 	c.Assert(obtained.Results, gc.HasLen, len(expected))
 	for i, one := range obtained.Results {
+		c.Assert(one.Error, gc.IsNil)
 		// We are only concerned with images here
 		c.Assert(one.Result.ImageMetadata, gc.DeepEquals, expected[i])
 	}

apiserver/facades/agent/provisioner/provisioninginfo.go

@@ -719,11 +719,15 @@ func (api *ProvisionerAPI) imageMetadataFromDataSources(env environs.Environ, co
 	for _, source := range sources {
 		api.logger.Debugf("looking in data source %v", source.Description())
 		found, info, err := imagemetadata.Fetch(fetcher, []simplestreams.DataSource{source}, constraint)
-		if err != nil {
+		if errors.Is(err, errors.NotFound) || errors.Is(err, errors.Unauthorized) {
 			// Do not stop looking in other data sources if there is an issue here.
 			api.logger.Warningf("encountered %v while getting published images metadata from %v", err, source.Description())
 			continue
+		} else if err != nil {
+			// When we get an actual protocol/unexpected error, we need to stop.
+			return nil, errors.Annotatef(err, "failed getting published images metadata from %s", source.Description())
 		}
+
 		for _, m := range found {
 			metadataState = append(metadataState, toModel(m, info.Source, source.Priority()))
 		}

cmd/juju/commands/main_test.go

@@ -235,6 +235,11 @@ func (s *MainSuite) TestNoWarn2xFirstRun(c *gc.C) {
 	path := c.MkDir()
 	s.PatchEnvironment("JUJU_HOME", path)
 
+	s.PatchValue(&cloud.FetchAndMaybeUpdatePublicClouds,
+		func(access cloud.PublicCloudsAccessDetails, updateClient bool) (map[string]jujucloud.Cloud, string, error) {
+			return nil, "", nil
+		})
+
 	var code int
 	stdout, stderr := jujutesting.CaptureOutput(c, func() {
 		code = jujuMain{

cmd/juju/status/status.go

@@ -383,29 +383,39 @@ func (c *statusCommand) runStatus(ctx *cmd.Context) error {
 	return nil
 }
 
-// statusArgsWithoutWatchFlag returns all args cut off '--watch' flag of status commands
-// and the value of '--watch' flag
-func (c *statusCommand) statusArgsWithoutWatchFlag(args []string) []string {
+// statusCommandForViddy returns the full juju command including all args
+// except the '--watch' flag.
+func (c *statusCommand) statusCommandForViddy(args []string) []string {
 	var jujuStatusArgsWithoutWatchFlag []string
 
 	for i := range args {
+		// In order to support gnu flags, we must first check if the
+		// watch flag is using gnu style. In that case, we must remove
+		// the entire arg, since it's one entire string (e.g.
+		// --watch=1s).
+		if strings.HasPrefix(args[i], "--watch=") {
+			jujuStatusArgsWithoutWatchFlag = append(args[:i], args[i+1:]...)
+			break
+		}
+		// If the flag is not using gnu style, we must remove both the
+		// flag and the argument (e.g --watch 1s)
 		if args[i] == "--watch" {
 			jujuStatusArgsWithoutWatchFlag = append(args[:i], args[i+2:]...)
-			if !c.noColor {
-				jujuStatusArgsWithoutWatchFlag = append(jujuStatusArgsWithoutWatchFlag, "--color")
-			}
 			break
 		}
 	}
 
+	if !c.noColor {
+		jujuStatusArgsWithoutWatchFlag = append(jujuStatusArgsWithoutWatchFlag, "--color")
+	}
 	return jujuStatusArgsWithoutWatchFlag
 }
 
 func (c *statusCommand) Run(ctx *cmd.Context) error {
 	defer c.close()
 
 	if c.watch != 0 {
-		jujuStatusArgs := c.statusArgsWithoutWatchFlag(os.Args)
+		jujuStatusArgs := c.statusCommandForViddy(os.Args)
 
 		viddyArgs := append([]string{"--no-title", "--interval", c.watch.String()}, jujuStatusArgs...)
 

cmd/juju/status/status_internal_test.go

@@ -5948,3 +5948,16 @@ controller  kontroll    dummy/dummy-region  1.2.3    unsupported  15:04:05+07:00
 	_, _, stderr = runStatus(c, ctx, "--no-color", "cannot", "match", "me")
 	c.Check(stderr, gc.Equals, "Nothing matched specified filters.\n")
 }
+
+func (s *StatusSuite) TestStatusArgsWithoutWatch(c *gc.C) {
+	cmd, err := initStatusCommand(s.store)
+	c.Assert(err, jc.ErrorIsNil)
+
+	statusArgsGNUStyle := []string{"juju", "status", "--watch=1s", "--relations"}
+	expectedArgsGNUStyle := []string{"juju", "status", "--relations", "--color"}
+	c.Check(cmd.statusCommandForViddy(statusArgsGNUStyle), jc.SameContents, expectedArgsGNUStyle)
+
+	statusArgsUnixStyle := []string{"juju", "status", "--watch", "1s", "--relations"}
+	expectedArgsUnixStyle := []string{"juju", "status", "--relations", "--color"}
+	c.Check(cmd.statusCommandForViddy(statusArgsUnixStyle), jc.SameContents, expectedArgsUnixStyle)
+}

core/base/supportedseries_test.go

@@ -264,16 +264,20 @@ func (s *SupportedSeriesSuite) TestUbuntuVersions(c *gc.C) {
 			WorkloadType: ControllerWorkloadType,
 			Version:      "23.04",
 		},
+		Mantic: {
+			WorkloadType: ControllerWorkloadType,
+			Version:      "23.10",
+		},
 	}
 
 	result := ubuntuVersions(nil, nil, ubuntuSeries)
-	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "bionic": "18.04", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "focal": "20.04", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "jammy": "22.04", "kinetic": "22.10", "lunar": "23.04", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "trusty": "14.04", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "xenial": "16.04", "yakkety": "16.10", "zesty": "17.04"})
+	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "bionic": "18.04", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "focal": "20.04", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "jammy": "22.04", "kinetic": "22.10", "lunar": "23.04", "mantic": "23.10", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "trusty": "14.04", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "xenial": "16.04", "yakkety": "16.10", "zesty": "17.04"})
 
 	result = ubuntuVersions(boolPtr(true), boolPtr(true), ubuntuSeries)
 	c.Check(result, gc.DeepEquals, map[string]string{"focal": "20.04", "jammy": "22.04"})
 
 	result = ubuntuVersions(boolPtr(false), boolPtr(false), ubuntuSeries)
-	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "yakkety": "16.10", "zesty": "17.04"})
+	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "mantic": "23.10", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "yakkety": "16.10", "zesty": "17.04"})
 
 	result = ubuntuVersions(boolPtr(true), boolPtr(false), ubuntuSeries)
 	c.Check(result, gc.DeepEquals, map[string]string{})
@@ -285,13 +289,13 @@ func (s *SupportedSeriesSuite) TestUbuntuVersions(c *gc.C) {
 	c.Check(result, gc.DeepEquals, map[string]string{"focal": "20.04", "jammy": "22.04"})
 
 	result = ubuntuVersions(boolPtr(false), nil, ubuntuSeries)
-	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "bionic": "18.04", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "trusty": "14.04", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "xenial": "16.04", "yakkety": "16.10", "zesty": "17.04"})
+	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "bionic": "18.04", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "mantic": "23.10", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "trusty": "14.04", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "xenial": "16.04", "yakkety": "16.10", "zesty": "17.04"})
 
 	result = ubuntuVersions(nil, boolPtr(true), ubuntuSeries)
 	c.Check(result, gc.DeepEquals, map[string]string{"bionic": "18.04", "focal": "20.04", "jammy": "22.04", "trusty": "14.04", "xenial": "16.04"})
 
 	result = ubuntuVersions(nil, boolPtr(false), ubuntuSeries)
-	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "yakkety": "16.10", "zesty": "17.04"})
+	c.Check(result, gc.DeepEquals, map[string]string{"artful": "17.10", "cosmic": "18.10", "disco": "19.04", "eoan": "19.10", "groovy": "20.10", "hirsute": "21.04", "impish": "21.10", "kinetic": "22.10", "lunar": "23.04", "mantic": "23.10", "precise": "12.04", "quantal": "12.10", "raring": "13.04", "saucy": "13.10", "utopic": "14.10", "vivid": "15.04", "wily": "15.10", "yakkety": "16.10", "zesty": "17.04"})
 }
 
 func makeTempFile(c *gc.C, content string) (*os.File, func()) {

environs/bootstrap/bootstrap.go

@@ -982,10 +982,13 @@ func bootstrapImageMetadata(
 	var publicImageMetadata []*imagemetadata.ImageMetadata
 	for _, source := range sources {
 		sourceMetadata, _, err := imagemetadata.Fetch(fetcher, []simplestreams.DataSource{source}, imageConstraint)
-		if err != nil {
+		if errors.Is(err, errors.NotFound) || errors.Is(err, errors.Unauthorized) {
 			logger.Debugf("ignoring image metadata in %s: %v", source.Description(), err)
 			// Just keep looking...
 			continue
+		} else if err != nil {
+			// When we get an actual protocol/unexpected error, we need to stop.
+			return nil, errors.Annotatef(err, "failed looking for image metadata in %s", source.Description())
 		}
 		logger.Debugf("found %d image metadata in %s", len(sourceMetadata), source.Description())
 		publicImageMetadata = append(publicImageMetadata, sourceMetadata...)
@@ -1136,7 +1139,7 @@ func setPrivateMetadataSources(fetcher imagemetadata.SimplestreamsFetcher, metad
 	}
 	existingMetadata, _, err := imagemetadata.Fetch(fetcher, []simplestreams.DataSource{dataSource}, imageConstraint)
 	if err != nil && !errors.Is(err, errors.NotFound) {
-		return nil, errors.Annotate(err, "cannot read image metadata")
+		return nil, errors.Annotatef(err, "cannot read image metadata in %s", dataSource.Description())
 	}
 
 	// Add an image metadata datasource for constraint validation, etc.

environs/bootstrap/bootstrap_test.go

@@ -6,6 +6,8 @@ package bootstrap_test
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"strings"
@@ -574,6 +576,12 @@ func (s *bootstrapSuite) setupProviderWithNoSupportedArches(c *gc.C) bootstrapEn
 // despite image metadata in other data sources compatible with the same configuration as well.
 // Related to bug#1560625.
 func (s *bootstrapSuite) TestBootstrapImageMetadataFromAllSources(c *gc.C) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	}))
+	defer server.Close()
+
+	s.PatchValue(&imagemetadata.DefaultUbuntuBaseURL, server.URL)
 	s.PatchValue(&osseries.HostSeries, func() (string, error) { return "raring", nil })
 	s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })