Merge pull request #12909 from wallyworld/merge-2.9-20210421

#12909

Merge 2.9

#12827 Unsubscribe from hub when closing state pool
#12829 Correct default bootstrap-timeout value displayed in help.
#12840 Constraint tags can be used for pod affinity
#12842 Fix upgrade series agent version handling
#12794 Add disk provisioning customization
#12845 Restore space support for manual machines
#12839 Support merging of netplan configs
#12853 Add display type for network-get results
#12854 Fix for LP1921557 sni in Juju login.
#12850 Use Base in Charmhub packge and its response structures.
#12858 Ensure assess-upgrade-series does not report started prematuremly
#12860 Removed logging from core annotations.
#12861 Fixes bug where empty error can happen in storage
#12865 Update Pebble version to include new files API
#12866 Workaround for k8s dashboard URL with k8s client proxy
#12862 Fix/lp 1923051
#12867 Fix/lp 1923561
#12870 Use channel logic in charm library
#12873 Add support for setting pod affinity topology key
#12874 Use Patch instead of Update for ensuring ingress resources
#12831 Integration fixes
#12879 Ensure refresh uses version
#12864 bug: fix for bootstrap fail on vsphere 7 + multi network
#12883 Initial work to allow juju trust for sidecar charms
#12884 Fix ssh with sidecar charms and containers.
#12886 Charmhub bases
#12881 Use charm pkg updates
#12889 Ignore projected volume mounts when looking up juju storage
#12890 Fix passing empty string container name to unit params
#12893 Add CLA checker GH action and remove codecov push action
#12897 Use production charmhub endpoint
#12887 Resource validation error
#12888 Ensure we validate the model target
#12898 Remove usage of systems package from CAAS application provisioner
#12899 CAAS bundle deployments
#12900 Bump up Pebble version to include user/group in list-files
#12901 charm Format helper
#12902 charm Iskubernetes helper
#12903 Display ... for really long k8s app versions in status
#12904 Filter out more full registry paths for app version in status
#12905 Fix k8s bundle deploys with v2 charms
#12906 Register resource-get for containeragent binary

Conflicts mostly due to charm v8 vs v9 imports.
The other one was due to changes to dashboard CLI.
```
# Conflicts:
# api/common/charms/common.go
# api/common/charms/common_test.go
# apiserver/facades/client/application/application.go
# apiserver/facades/client/application/charmstore.go
# apiserver/facades/client/application/update_series_mocks_test.go
# apiserver/facades/client/charms/client.go
# apiserver/facades/client/charms/convertions.go
# apiserver/facades/client/machinemanager/types_mock_test.go
# apiserver/facades/controller/caasoperatorprovisioner/provisioner.go
# cmd/juju/application/deployer/bundlehandler_test.go
# cmd/juju/application/refresh_test.go
# cmd/juju/application/refresher/refresher_mock_test.go
# cmd/juju/dashboard/dashboard.go
# core/charm/strategies_mock_test.go
# core/model/model.go
# core/model/model_test.go
# go.mod
# go.sum
# resource/resourceadapters/charmhub.go
# scripts/win-installer/setup.iss
# service/agentconf_test.go
# snap/snapcraft.yaml
# state/charm.go
# state/migration_export.go
# state/state.go
# version/version.go
# worker/caasfirewallerembedded/applicationworker.go
# worker/caasfirewallerembedded/applicationworker_test.go
```

## QA steps

See PRs

.github/workflows/microk8s-tests.yml

@@ -53,6 +53,17 @@ jobs:
         # enable now to give microk8s more time to settle down.
         addons: '["dns", "storage", "dashboard", "ingress", "metallb:10.64.140.43-10.64.140.49"]'
 
+    # Remove once this is fixed:
+    # https://github.com/actions/virtual-environments/issues/3185
+    - name: Twiddle with MicroK8s
+      run: |
+        set -eux
+        echo '--kubelet-preferred-address-types="InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"' | sudo tee -a /var/snap/microk8s/current/args/kube-apiserver
+        sudo snap restart microk8s
+        sg microk8s -c 'microk8s status --wait-ready'
+        kubectl -nkube-system rollout status ds/calico-node
+        kubectl wait --for=condition=available -nkube-system deployment/coredns deployment/hostpath-provisioner
+
     - name: Install Dependencies
       run: |
         set -eux
@@ -83,8 +94,6 @@ jobs:
           set -eux
           git clone https://github.com/juju-solutions/bundle-kubeflow.git
           cd bundle-kubeflow
-          # TODO: tmp fix, remove me later once current kubeflow master branch published.
-          git reset --hard 5e0b6fcb
           git clone git://git.launchpad.net/canonical-osm
           cp -r canonical-osm/charms/interfaces/juju-relation-mysql mysql
 

acceptancetests/assess_upgrade_series.py

@@ -20,6 +20,7 @@
     add_basic_testing_arguments,
     configure_logging,
     JujuAssertionError,
+    wait_for_port,
 )
 
 __metaclass__ = type
@@ -81,8 +82,9 @@ def reboot_machine(client, machine):
             raise e
         log.info("Ignoring `juju ssh` exit status after triggering reboot")
 
-    log.info("wait_for_started()")
-    client.wait_for_started()
+    log.info("waiting for reboot")
+    hostname = client.get_status().get_machine_dns_name(machine)
+    wait_for_port(hostname, 22, timeout=600)
 
 
 def assert_correct_series(client, machine, expected):

acceptancetests/jujupy/client.py

@@ -1760,7 +1760,7 @@ def get_controller_member_status(info_dict):
         return info_dict.get('controller-member-status')
 
     def wait_for_ha(self, timeout=1200, start=None, quorum=3):
-        """Wait for voiting to be enabled.
+        """Wait for voting to be enabled.
 
         May only be called on a controller client."""
         if self.env.environment != self.get_controller_model_name():
@@ -1784,7 +1784,7 @@ def status_to_ha(status):
         self._wait_for_status(reporter, status_to_ha, VotingNotEnabled,
                               timeout=timeout, start=start)
         # XXX sinzui 2014-12-04: bug 1399277 happens because
-        # juju claims HA is ready when the monogo replica sets
+        # juju claims HA is ready when the mongo replica sets
         # are not. Juju is not fully usable. The replica set
         # lag might be 5 minutes.
         self._backend.pause(300)

api/caasfirewaller/client_test.go

@@ -4,6 +4,7 @@
 package caasfirewaller_test
 
 import (
+	"github.com/juju/charm/v9"
 	"github.com/juju/errors"
 	"github.com/juju/names/v4"
 	"github.com/juju/testing"
@@ -124,6 +125,7 @@ func (s *firewallerEmbeddedSuite) TestApplicationCharmInfo(c *gc.C) {
 	c.Assert(result, jc.DeepEquals, &apicommoncharms.CharmInfo{
 		Revision: 1,
 		URL:      `cs:gitlab-0`,
+		Manifest: &charm.Manifest{},
 	})
 }
 

api/caasunitprovisioner/client.go

@@ -120,6 +120,45 @@ func (c *Client) ApplicationScale(applicationName string) (int, error) {
 	return results.Results[0].Result, nil
 }
 
+// ApplicationTrust returns the trust value for the specified application.
+func (c *Client) ApplicationTrust(applicationName string) (bool, error) {
+	var results params.BoolResults
+	args := params.Entities{
+		Entities: []params.Entity{{Tag: names.NewApplicationTag(applicationName).String()}},
+	}
+	err := c.facade.FacadeCall("ApplicationsTrust", args, &results)
+	if err != nil {
+		return false, errors.Trace(err)
+	}
+	if len(results.Results) != len(args.Entities) {
+		return false, errors.Errorf("expected %d result(s), got %d", len(args.Entities), len(results.Results))
+	}
+	return results.Results[0].Result, nil
+}
+
+// WatchApplicationTrustHash returns a StringsWatcher that notifies of
+// changes to the application's trust hash.
+func (c *Client) WatchApplicationTrustHash(application string) (watcher.StringsWatcher, error) {
+	applicationTag, err := applicationTag(application)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	args := entities(applicationTag)
+
+	var results params.StringsWatchResults
+	if err := c.facade.FacadeCall("WatchApplicationsTrustHash", args, &results); err != nil {
+		return nil, err
+	}
+	if n := len(results.Results); n != 1 {
+		return nil, errors.Errorf("expected 1 result, got %d", n)
+	}
+	if err := results.Results[0].Error; err != nil {
+		return nil, errors.Trace(err)
+	}
+	w := apiwatcher.NewStringsWatcher(c.facade.RawAPICaller(), results.Results[0])
+	return w, nil
+}
+
 // DeploymentMode returns the deployment mode for the specified application's charm.
 func (c *Client) DeploymentMode(applicationName string) (caas.DeploymentMode, error) {
 	var results params.StringResults

api/caasunitprovisioner/client_test.go

@@ -512,3 +512,55 @@ func (s *unitprovisionerSuite) TestClearApplicationResources(c *gc.C) {
 	err := client.ClearApplicationResources("gitlab")
 	c.Assert(err, gc.ErrorMatches, "FAIL")
 }
+
+func (s *unitprovisionerSuite) TestWatchApplicationTrustHash(c *gc.C) {
+	apiCaller := basetesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
+		c.Check(objType, gc.Equals, "CAASUnitProvisioner")
+		c.Check(version, gc.Equals, 0)
+		c.Check(id, gc.Equals, "")
+		c.Check(request, gc.Equals, "WatchApplicationsTrustHash")
+		c.Assert(arg, jc.DeepEquals, params.Entities{
+			Entities: []params.Entity{{
+				Tag: "application-gitlab",
+			}},
+		})
+		c.Assert(result, gc.FitsTypeOf, &params.StringsWatchResults{})
+		*(result.(*params.StringsWatchResults)) = params.StringsWatchResults{
+			Results: []params.StringsWatchResult{{
+				Error: &params.Error{Message: "FAIL"},
+			}},
+		}
+		return nil
+	})
+
+	client := caasunitprovisioner.NewClient(apiCaller)
+	watcher, err := client.WatchApplicationTrustHash("gitlab")
+	c.Assert(watcher, gc.IsNil)
+	c.Assert(err, gc.ErrorMatches, "FAIL")
+}
+
+func (s *unitprovisionerSuite) TestApplicationTrust(c *gc.C) {
+	apiCaller := basetesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
+		c.Check(objType, gc.Equals, "CAASUnitProvisioner")
+		c.Check(version, gc.Equals, 0)
+		c.Check(id, gc.Equals, "")
+		c.Check(request, gc.Equals, "ApplicationsTrust")
+		c.Assert(arg, jc.DeepEquals, params.Entities{
+			Entities: []params.Entity{{
+				Tag: "application-gitlab",
+			}},
+		})
+		c.Assert(result, gc.FitsTypeOf, &params.BoolResults{})
+		*(result.(*params.BoolResults)) = params.BoolResults{
+			Results: []params.BoolResult{{
+				Result: true,
+			}},
+		}
+		return nil
+	})
+
+	client := caasunitprovisioner.NewClient(apiCaller)
+	trust, err := client.ApplicationTrust("gitlab")
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(trust, jc.IsTrue)
+}

api/common/charm/charmorigin.go

@@ -4,6 +4,8 @@
 package charm
 
 import (
+	"github.com/juju/charm/v9"
+
 	"github.com/juju/juju/apiserver/params"
 	corecharm "github.com/juju/juju/core/charm"
 )
@@ -55,15 +57,15 @@ func (o Origin) WithSeries(series string) Origin {
 	return other
 }
 
-// CoreChannel returns the core charm channel.
-func (o Origin) CoreChannel() corecharm.Channel {
+// CharmChannel returns the the channel indicated by this origin.
+func (o Origin) CharmChannel() charm.Channel {
 	var track string
 	if o.Track != nil {
 		track = *o.Track
 	}
-	return corecharm.Channel{
+	return charm.Channel{
 		Track: track,
-		Risk:  corecharm.Risk(o.Risk),
+		Risk:  charm.Risk(o.Risk),
 	}
 }
 
@@ -90,10 +92,10 @@ func (o Origin) CoreCharmOrigin() corecharm.Origin {
 	if o.Track != nil {
 		track = *o.Track
 	}
-	var channel *corecharm.Channel
+	var channel *charm.Channel
 	if o.Risk != "" {
-		channel = &corecharm.Channel{
-			Risk:  corecharm.Risk(o.Risk),
+		channel = &charm.Channel{
+			Risk:  charm.Risk(o.Risk),
 			Track: track,
 		}
 	}
@@ -132,7 +134,7 @@ func APICharmOrigin(origin params.CharmOrigin) Origin {
 // CoreCharmOrigin is a helper function to convert params.CharmOrigin
 // to an Origin.
 func CoreCharmOrigin(origin corecharm.Origin) Origin {
-	var ch corecharm.Channel
+	var ch charm.Channel
 	if origin.Channel != nil {
 		ch = *origin.Channel
 	}

api/common/charm/charmorigin_test.go

@@ -4,9 +4,10 @@
 package charm_test
 
 import (
-	"github.com/juju/juju/api/common/charm"
-	corecharm "github.com/juju/juju/core/charm"
+	"github.com/juju/charm/v9"
 	gc "gopkg.in/check.v1"
+
+	commoncharm "github.com/juju/juju/api/common/charm"
 )
 
 type originSuite struct{}
@@ -15,26 +16,26 @@ var _ = gc.Suite(&originSuite{})
 
 func (originSuite) TestCoreChannel(c *gc.C) {
 	track := "latest"
-	origin := charm.Origin{
+	origin := commoncharm.Origin{
 		Risk:  "edge",
 		Track: &track,
 	}
-	c.Assert(origin.CoreChannel(), gc.DeepEquals, corecharm.Channel{
-		Risk:  corecharm.Edge,
+	c.Assert(origin.CharmChannel(), gc.DeepEquals, charm.Channel{
+		Risk:  charm.Edge,
 		Track: "latest",
 	})
 }
 
 func (originSuite) TestCoreChannelWithEmptyTrack(c *gc.C) {
-	origin := charm.Origin{
+	origin := commoncharm.Origin{
 		Risk: "edge",
 	}
-	c.Assert(origin.CoreChannel(), gc.DeepEquals, corecharm.Channel{
-		Risk: corecharm.Edge,
+	c.Assert(origin.CharmChannel(), gc.DeepEquals, charm.Channel{
+		Risk: charm.Edge,
 	})
 }
 
 func (originSuite) TestCoreChannelThatIsEmpty(c *gc.C) {
-	origin := charm.Origin{}
-	c.Assert(origin.CoreChannel(), gc.DeepEquals, corecharm.Channel{})
+	origin := commoncharm.Origin{}
+	c.Assert(origin.CharmChannel(), gc.DeepEquals, charm.Channel{})
 }

api/common/charms/common.go

@@ -5,11 +5,11 @@
 package charms
 
 import (
+	"fmt"
+
 	"github.com/juju/charm/v9"
 	"github.com/juju/charm/v9/resource"
 	"github.com/juju/errors"
-	"github.com/juju/systems"
-	"github.com/juju/systems/channel"
 	"github.com/juju/version/v2"
 
 	"github.com/juju/juju/api/base"
@@ -34,6 +34,7 @@ type CharmInfo struct {
 	Meta       *charm.Meta
 	Actions    *charm.Actions
 	Metrics    *charm.Metrics
+	Manifest   *charm.Manifest
 	LXDProfile *charm.LXDProfile
 }
 
@@ -52,13 +53,18 @@ func (c *CharmsClient) CharmInfo(charmURL string) (*CharmInfo, error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
+	manifest, err := convertCharmManifest(info.Manifest)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
 	result := &CharmInfo{
 		Revision:   info.Revision,
 		URL:        info.URL,
 		Config:     params.FromCharmOptionMap(info.Config),
 		Meta:       meta,
 		Actions:    convertCharmActions(info.Actions),
 		Metrics:    convertCharmMetrics(info.Metrics),
+		Manifest:   manifest,
 		LXDProfile: convertCharmLXDProfile(info.LXDProfile),
 	}
 	return result, nil
@@ -76,10 +82,6 @@ func convertCharmMeta(meta *params.CharmMeta) (*charm.Meta, error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	bases, err := convertCharmBases(meta.Bases)
-	if err != nil {
-		return nil, errors.Trace(err)
-	}
 	containers, err := convertCharmContainers(meta.Containers)
 	if err != nil {
 		return nil, errors.Trace(err)
@@ -103,7 +105,6 @@ func convertCharmMeta(meta *params.CharmMeta) (*charm.Meta, error) {
 		Resources:      resources,
 		Terms:          meta.Terms,
 		MinJujuVersion: minVersion,
-		Bases:          bases,
 		Containers:     containers,
 		Assumes:        meta.Assumes,
 	}
@@ -350,6 +351,10 @@ func (c *charmImpl) Metrics() *charm.Metrics {
 	return c.info.Metrics
 }
 
+func (c *charmImpl) Manifest() *charm.Manifest {
+	return c.info.Manifest
+}
+
 func (c *charmImpl) Actions() *charm.Actions {
 	return c.info.Actions
 }
@@ -358,23 +363,20 @@ func (c *charmImpl) Revision() int {
 	return c.info.Revision
 }
 
-func convertCharmBases(input []params.CharmBase) ([]systems.Base, error) {
-	var err error
-	res := []systems.Base(nil)
-	for _, v := range input {
-		ch := channel.Channel{}
-		if v.Channel != "" {
-			ch, err = channel.Parse(v.Channel)
-			if err != nil {
-				return nil, errors.Trace(err)
-			}
+func convertCharmManifest(input *params.CharmManifest) (*charm.Manifest, error) {
+	if input == nil {
+		return &charm.Manifest{}, nil
+	}
+	res := []charm.Base(nil)
+	for _, v := range input.Bases {
+		str := fmt.Sprintf("%s/%s", v.Name, v.Channel)
+		b, err := charm.ParseBase(str, v.Architectures...)
+		if err != nil {
+			return nil, errors.Trace(err)
 		}
-		res = append(res, systems.Base{
-			Name:    v.Name,
-			Channel: ch,
-		})
+		res = append(res, b)
 	}
-	return res, nil
+	return &charm.Manifest{Bases: res}, nil
 }
 
 func convertCharmContainers(input map[string]params.CharmContainer) (map[string]charm.Container, error) {