Merge pull request #15717 from wallyworld/merge-3.1-20230608

#15717

Merge 3.1

#15676 [from wallyworld/newer-clients-migrate](7c1f884)
#15672 [from hpidcock/bump-juju-description-v3.0.15](acec126)
#15683 [from hpidcock/fix-persistent-storage-test](460dd21)
#15673 [from barrettj12/check-merge](5c253c3)
#15677 [from barrettj12/invalid-offer](3cb3f8b)
#15654 [from ycliuhw/fix/backendRefCount](4e5ae3c)
#15692 [from wallyworld/fix-secrets-cmr](a1fb0c4)
[t](840bc09) #15709 [from wallyworld/hook-secret-revison](840bc09)
#15701 [from hpidcock/fix-upgrade-podspec-sidecar](d465c93)
#15681 [from anvial/JUJU-3882-fix-test-deploy-test-…](f54c1ad)
#15714 [from wallyworld/offer-consume=sameapp](6390036)

Conflicts were upgrade steps - the 3.1.3 step has been moved to 3.2.1.
Also an auth tweak to crossmodelrelaltions.

```
# Conflicts:
# apiserver/common/crossmodel/auth_test.go
# apiserver/facades/controller/crossmodelrelations/crossmodelrelations.go
# rpc/params/apierror.go
# state/upgrades.go
# state/upgrades_test.go
# upgrades/backend.go
#
```

## QA steps

See PRs

.github/workflows/merge.yml

@@ -0,0 +1,82 @@
+name: Merge
+on:
+  push:
+    branches: ['2.9', '3.1', '3.2', '3.3']
+
+jobs:
+  check-merge:
+    name: Check for conflicts
+    runs-on: ubuntu-latest
+    env:
+      MERGE_TARGETS: |
+        2.9: 3.1
+        3.1: 3.2
+        3.2: 3.3
+        3.3: main
+
+    steps:
+      - name: Determine source/target branches
+        id: branch
+        run: |
+          set -x
+          SOURCE_BRANCH=${{ github.ref_name }}
+          echo "source=$SOURCE_BRANCH" >> "$GITHUB_OUTPUT"
+          TARGET_BRANCH=$(echo "$MERGE_TARGETS" | yq ".\"$SOURCE_BRANCH\"")
+          echo "target=$TARGET_BRANCH" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.branch.outputs.target }}
+
+      - name: Attempt to merge
+        id: merge
+        run: |
+          set -x
+          # Need to set Git username/email to do the merge (yawn)
+          git config user.name 'jujubot'
+          git config user.email 'fake@address.me'
+          
+          set +e
+          git merge origin/${{ steps.branch.outputs.source }}
+          case $? in
+          0)
+            echo "conflicts=false" >> "$GITHUB_OUTPUT"
+            ;;
+          1)
+            echo "conflicts=true" >> "$GITHUB_OUTPUT"
+            ;;
+          *)
+            exit $?
+            ;;
+          esac
+
+      - name: Notify if merge has conflicts
+        if: steps.merge.outputs.conflicts == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MM_TOKEN: ${{ secrets.MM_TOKEN }}
+          MM_USERS: ${{ secrets.MM_USERS }}
+        run: |
+          set -ex
+          SOURCE_BRANCH=${{ steps.branch.outputs.source }}
+          TARGET_BRANCH=${{ steps.branch.outputs.target }}
+          
+          # Get PR info
+          PR_INFO=$(gh pr list --search="${{ github.sha }}" --state=merged --base="$SOURCE_BRANCH" --json='number,author')
+          PR_NUMBER=$(echo "$PR_INFO" | jq '.[].number')
+          
+          PR_AUTHOR=$(echo "$PR_INFO" | jq -r '.[].author.login')
+          MM_USER=$(echo "$MM_USERS" | jq -r ".\"$PR_AUTHOR\"")
+          if [[ $MM_USER == '' || $MM_USER == null ]]; then
+            MM_USER=$PR_AUTHOR
+          fi
+          
+          MESSAGE="@$MM_USER your PR [#$PR_NUMBER](https://github.com/juju/juju/pull/$PR_NUMBER) has created merge conflicts - please merge $SOURCE_BRANCH into $TARGET_BRANCH and resolve the conflicts. Thanks! :)"
+          
+          # install mmctl
+          curl https://github.com/mattermost/mmctl/releases/download/v7.8.5/linux_amd64.tar -Lo mmctl.tar
+          tar -xvf mmctl.tar
+          ./mmctl auth login 'https://chat.charmhub.io' --name Charmhub --access-token $MM_TOKEN
+          ./mmctl post create Charmhub:juju-watercooler --message "$MESSAGE"

api/apiclient.go

@@ -1237,52 +1237,41 @@ func isX509Error(err error) bool {
 	}
 }
 
-var apiCallRetryStrategy = retry.LimitTime(10*time.Second,
-	retry.Exponential{
-		Initial:  100 * time.Millisecond,
-		Factor:   2,
-		MaxDelay: 1500 * time.Millisecond,
-	},
-)
-
 // APICall places a call to the remote machine.
 //
 // This fills out the rpc.Request on the given facade, version for a given
 // object id, and the specific RPC method. It marshalls the Arguments, and will
 // unmarshall the result into the response object that is supplied.
 func (s *state) APICall(facade string, vers int, id, method string, args, response interface{}) error {
-	for a := retry.Start(apiCallRetryStrategy, s.clock); a.Next(); {
-		err := s.client.Call(rpc.Request{
-			Type:    facade,
-			Version: vers,
-			Id:      id,
-			Action:  method,
-		}, args, response)
-		if err == nil {
-			return nil
-		}
-		code := params.ErrCode(err)
-		if code != params.CodeIncompatibleClient {
-			return errors.Trace(err)
-		}
-		// Default to major version 2 for older servers.
-		serverMajorVersion := 2
-		err = errors.Cause(err)
-		apiErr, ok := err.(*rpc.RequestError)
-		if ok {
-			if serverVersion, ok := apiErr.Info["server-version"]; ok {
-				serverVers, err := version.Parse(fmt.Sprintf("%v", serverVersion))
-				if err == nil {
-					serverMajorVersion = serverVers.Major
-				}
+	err := s.client.Call(rpc.Request{
+		Type:    facade,
+		Version: vers,
+		Id:      id,
+		Action:  method,
+	}, args, response)
+	if err == nil {
+		return nil
+	}
+	code := params.ErrCode(err)
+	if code != params.CodeIncompatibleClient {
+		return errors.Trace(err)
+	}
+	// Default to major version 2 for older servers.
+	serverMajorVersion := 2
+	err = errors.Cause(err)
+	apiErr, ok := err.(*rpc.RequestError)
+	if ok {
+		if serverVersion, ok := apiErr.Info["server-version"]; ok {
+			serverVers, err := version.Parse(fmt.Sprintf("%v", serverVersion))
+			if err == nil {
+				serverMajorVersion = serverVers.Major
 			}
 		}
-		logger.Debugf("%v.%v API call not supported", facade, method)
-		return errors.NewNotSupported(nil, fmt.Sprintf(
-			"juju client with major version %d used with a controller having major version %d not supported\nupdate your juju client to match the version running on the controller",
-			jujuversion.Current.Major, serverMajorVersion))
 	}
-	panic("unreachable")
+	logger.Debugf("%v.%v API call not supported", facade, method)
+	return errors.NewNotSupported(nil, fmt.Sprintf(
+		"juju client with version %d.%d used with a controller having major version %d not supported\nre-install your juju client to match the version running on the controller",
+		jujuversion.Current.Major, jujuversion.Current.Minor, serverMajorVersion))
 }
 
 func (s *state) Close() error {

api/apiclient_test.go

@@ -1207,8 +1207,8 @@ func (s *apiclientSuite) TestLoginIncompatibleClient(c *gc.C) {
 	err := conn.APICall("facade", 1, "id", "method", nil, nil)
 	c.Check(clock.waits, gc.HasLen, 0)
 	c.Assert(err, gc.ErrorMatches, fmt.Sprintf(
-		"juju client with major version %d used with a controller having major version %d not supported\\n.*",
-		jujuversion.Current.Major, 99,
+		"juju client with version %d.%d used with a controller having major version %d not supported\\n.*",
+		jujuversion.Current.Major, jujuversion.Current.Minor, 99,
 	))
 }
 

api/controller/crossmodelrelations/crossmodelrelations.go

@@ -465,20 +465,21 @@ func (c *Client) WatchOfferStatus(arg params.OfferArg) (watcher.OfferStatusWatch
 
 // WatchConsumedSecretsChanges returns a watcher which notifies of new secret revisions consumed by the
 // app with the specified token.
-func (c *Client) WatchConsumedSecretsChanges(applicationToken string, mac *macaroon.Macaroon) (watcher.SecretsRevisionWatcher, error) {
+func (c *Client) WatchConsumedSecretsChanges(applicationToken, relationToken string, mac *macaroon.Macaroon) (watcher.SecretsRevisionWatcher, error) {
 	var macs macaroon.Slice
 	if mac != nil {
 		macs = macaroon.Slice{mac}
 	}
 
 	args := params.WatchRemoteSecretChangesArgs{Args: []params.WatchRemoteSecretChangesArg{{
 		ApplicationToken: applicationToken,
+		RelationToken:    relationToken,
 		Macaroons:        macs,
 		BakeryVersion:    bakery.LatestVersion,
 	}}}
 
 	// Use any previously cached discharge macaroons.
-	if ms, ok := c.getCachedMacaroon("watch consumed secret changes", applicationToken); ok {
+	if ms, ok := c.getCachedMacaroon("watch consumed secret changes", relationToken); ok {
 		args.Args[0].Macaroons = ms
 		args.Args[0].BakeryVersion = bakery.LatestVersion
 	}
@@ -510,7 +511,7 @@ func (c *Client) WatchConsumedSecretsChanges(applicationToken string, mac *macar
 			return nil, result.Error
 		}
 		args.Args[0].Macaroons = mac
-		c.cache.Upsert(applicationToken, mac)
+		c.cache.Upsert(relationToken, mac)
 
 		if err := apiCall(); err != nil {
 			return nil, errors.Trace(err)

api/controller/crossmodelrelations/crossmodelrelations_test.go

@@ -715,7 +715,8 @@ func (s *CrossModelRelationsSuite) TestWatchOfferStatusDischargeRequired(c *gc.C
 }
 
 func (s *CrossModelRelationsSuite) TestWatchConsumedSecretsChanges(c *gc.C) {
-	appToken := "token"
+	appToken := "app-token"
+	relToken := "rel-token"
 	mac, err := apitesting.NewMacaroon("id")
 	c.Assert(err, jc.ErrorIsNil)
 	var callCount int
@@ -724,7 +725,7 @@ func (s *CrossModelRelationsSuite) TestWatchConsumedSecretsChanges(c *gc.C) {
 		c.Check(version, gc.Equals, 0)
 		c.Check(id, gc.Equals, "")
 		c.Check(arg, jc.DeepEquals, params.WatchRemoteSecretChangesArgs{Args: []params.WatchRemoteSecretChangesArg{{
-			ApplicationToken: appToken, Macaroons: macaroon.Slice{mac},
+			ApplicationToken: appToken, RelationToken: relToken, Macaroons: macaroon.Slice{mac},
 			BakeryVersion: bakery.LatestVersion,
 		}}})
 		c.Check(request, gc.Equals, "WatchConsumedSecretsChanges")
@@ -738,14 +739,14 @@ func (s *CrossModelRelationsSuite) TestWatchConsumedSecretsChanges(c *gc.C) {
 		return nil
 	})
 	client := crossmodelrelations.NewClientWithCache(apiCaller, s.cache)
-	_, err = client.WatchConsumedSecretsChanges(appToken, mac)
+	_, err = client.WatchConsumedSecretsChanges(appToken, relToken, mac)
 	c.Check(err, gc.ErrorMatches, "FAIL")
 	// Call again with a different macaroon but the first one will be
 	// cached and override the passed in macaroon.
 	different, err := apitesting.NewMacaroon("different")
 	c.Assert(err, jc.ErrorIsNil)
-	s.cache.Upsert(appToken, macaroon.Slice{mac})
-	_, err = client.WatchConsumedSecretsChanges(appToken, different)
+	s.cache.Upsert(relToken, macaroon.Slice{mac})
+	_, err = client.WatchConsumedSecretsChanges(appToken, relToken, different)
 	c.Check(err, gc.ErrorMatches, "FAIL")
 	c.Check(callCount, gc.Equals, 2)
 }
@@ -782,13 +783,13 @@ func (s *CrossModelRelationsSuite) TestWatchConsumedSecretsChangesDischargeRequi
 	acquirer := &mockDischargeAcquirer{}
 	callerWithBakery := testing.APICallerWithBakery(apiCaller, acquirer)
 	client := crossmodelrelations.NewClientWithCache(callerWithBakery, s.cache)
-	_, err := client.WatchConsumedSecretsChanges("token", nil)
+	_, err := client.WatchConsumedSecretsChanges("app-token", "rel-token", nil)
 	c.Check(callCount, gc.Equals, 2)
 	c.Check(err, jc.ErrorIsNil)
 	c.Assert(dischargeMac, gc.HasLen, 1)
 	c.Assert(dischargeMac[0].Id(), jc.DeepEquals, []byte("discharge mac"))
 	// Macaroon has been cached.
-	ms, ok := s.cache.Get("token")
+	ms, ok := s.cache.Get("rel-token")
 	c.Assert(ok, jc.IsTrue)
 	apitesting.MacaroonEquals(c, ms[0], dischargeMac[0])
 }

api/watcher/watcher_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-macaroon-bakery/macaroon-bakery/v3/bakery"
 	"github.com/go-macaroon-bakery/macaroon-bakery/v3/bakery/checkers"
+	"github.com/juju/charm/v10"
 	"github.com/juju/names/v4"
 	jc "github.com/juju/testing/checkers"
 	"github.com/juju/utils/v3"
@@ -722,14 +723,6 @@ func (s *watcherSuite) TestSecretsExpiryWatcher(c *gc.C) {
 func (s *watcherSuite) setupSecretsRevisionWatcher(
 	c *gc.C,
 ) (*secrets.URI, func(uri *secrets.URI, rev int), func(), func()) {
-	remoteApp, err := s.State.AddRemoteApplication(state.AddRemoteApplicationParams{
-		Name: "foo", OfferUUID: "offer-uuid", URL: "me/model.foo", SourceModel: s.Model.ModelTag()})
-	c.Assert(err, jc.ErrorIsNil)
-	// Export the remoteApp so it can be found with a token.
-	re := s.State.RemoteEntities()
-	token, err := re.ExportLocalEntity(remoteApp.Tag())
-	c.Assert(err, jc.ErrorIsNil)
-
 	// Set up the offer.
 	app := s.AddTestingApplication(c, "mysql", s.AddTestingCharm(c, "mysql"))
 	unit, password := s.Factory.MakeUnitReturningPassword(c, &factory.UnitParams{
@@ -771,6 +764,31 @@ func (s *watcherSuite) setupSecretsRevisionWatcher(
 		})
 	c.Assert(err, jc.ErrorIsNil)
 
+	remoteApp, err := s.State.AddRemoteApplication(state.AddRemoteApplicationParams{
+		Name: "foo", OfferUUID: offer.OfferUUID, URL: "me/model.foo", SourceModel: s.Model.ModelTag()})
+	c.Assert(err, jc.ErrorIsNil)
+	remoteRel, err := s.State.AddRelation(
+		state.Endpoint{"mysql", charm.Relation{Name: "server", Interface: "mysql", Role: charm.RoleProvider, Scope: charm.ScopeGlobal}},
+		state.Endpoint{"foo", charm.Relation{Name: "db", Interface: "mysql", Role: charm.RoleRequirer, Scope: charm.ScopeGlobal}},
+	)
+	c.Assert(err, jc.ErrorIsNil)
+
+	_, err = s.State.AddOfferConnection(state.AddOfferConnectionParams{
+		SourceModelUUID: utils.MustNewUUID().String(),
+		OfferUUID:       offer.OfferUUID,
+		Username:        "fred",
+		RelationId:      remoteRel.Id(),
+		RelationKey:     remoteRel.Tag().Id(),
+	})
+	c.Assert(err, jc.ErrorIsNil)
+
+	// Export the remote entities so they can be found with a token.
+	re := s.State.RemoteEntities()
+	appToken, err := re.ExportLocalEntity(remoteApp.Tag())
+	c.Assert(err, jc.ErrorIsNil)
+	relToken, err := re.ExportLocalEntity(remoteRel.Tag())
+	c.Assert(err, jc.ErrorIsNil)
+
 	// Create a secret to watch.
 	store := state.NewSecrets(s.State)
 	uri := secrets.NewURI()
@@ -795,7 +813,7 @@ func (s *watcherSuite) setupSecretsRevisionWatcher(
 	c.Assert(err, jc.ErrorIsNil)
 
 	client := crossmodelrelations.NewClient(apiConn)
-	w, err := client.WatchConsumedSecretsChanges(token, mac.M())
+	w, err := client.WatchConsumedSecretsChanges(appToken, relToken, mac.M())
 	if !c.Check(err, jc.ErrorIsNil) {
 		_ = apiConn.Close()
 		c.FailNow()