Merge pull request #13430 from wallyworld/merge-2.9-20211019

#13430

Merge 2.9

#13419 Recognise VXLAN network devices
#13423 Fix add-machine command help for zones constraint
#13422 Provision OpenStack instances with NICs for bindings
#13427 Update to latest Pebble
#13429 Fix lease manager next check scheduling
#13428 Various txn fixes mainly for mongo 4.4 compatibility

Conflicts due to mongo changes already in develop
```
# Conflicts:
# cloudconfig/podcfg/image.go
# cloudconfig/podcfg/podcfg.go
# featuretests/upgrade_test.go
# mongo/mongo.go
# testing/mgo.go
#
```

## QA steps

See PRs

.github/golangci-lint.config.experimental.yaml

@@ -6,7 +6,7 @@ linters:
   enable:
     - errcheck
 run:
-  timeout: 20m
+  timeout: 30m
   tests: false
 skip-dirs:
   - acceptancetests

.github/golangci-lint.config.yaml

@@ -35,6 +35,6 @@ linters:
     - deadcode
     - varcheck
 run:
-  timeout: 20m
+  timeout: 30m
 skip-dirs:
   - acceptancetests

acceptancetests/jujupy/controller.py

@@ -91,4 +91,4 @@ def mongo_memory_profile(self):
     def db_snap_channel(self):
         if 'juju-db-snap-channel' in self.cfg:
             return self.cfg["juju-db-snap-channel"]
-        return "4.0/stable"
+        return "4.4/stable"

agent/agent_test.go

@@ -385,7 +385,7 @@ var attributeParams = agent.AgentConfigParams{
 	Nonce:                    "a nonce",
 	Controller:               testing.ControllerTag,
 	Model:                    testing.ModelTag,
-	JujuDBSnapChannel:        "4.0/stable",
+	JujuDBSnapChannel:        controller.DefaultJujuDBSnapChannel,
 	NonSyncedWritesToRaftLog: false,
 }
 
@@ -400,7 +400,7 @@ func (*suite) TestAttributes(c *gc.C) {
 	c.Assert(conf.Dir(), gc.Equals, "/data/dir/agents/machine-1")
 	c.Assert(conf.Nonce(), gc.Equals, "a nonce")
 	c.Assert(conf.UpgradedToVersion(), jc.DeepEquals, jujuversion.Current)
-	c.Assert(conf.JujuDBSnapChannel(), gc.Equals, "4.0/stable")
+	c.Assert(conf.JujuDBSnapChannel(), gc.Equals, "4.4/stable")
 	c.Assert(conf.NonSyncedWritesToRaftLog(), jc.IsFalse)
 }
 

apiserver/facades/agent/provisioner/provisioninginfo.go

@@ -341,8 +341,9 @@ func (api *ProvisionerAPI) machineSpaces(m *state.Machine,
 func (api *ProvisionerAPI) machineSpaceTopology(machineID string, spaceNames []string) (params.ProvisioningNetworkTopology, error) {
 	var topology params.ProvisioningNetworkTopology
 
-	// If there are no space names, or if there is only one space name and
-	// that's the alpha space.
+	// If there are no space names, or if there is only one space
+	// name and that's the alpha space, we don't bother setting a
+	// topology that constrains provisioning.
 	if len(spaceNames) < 1 || (len(spaceNames) == 1 && spaceNames[0] == network.AlphaSpaceName) {
 		return topology, nil
 	}

apiserver/facades/controller/caasapplicationprovisioner/state.go

@@ -29,7 +29,7 @@ type CAASApplicationProvisionerState interface {
 	WatchApplications() state.StringsWatcher
 }
 
-// CAASApplicationProvisionerState provides the subset of controller state
+// CAASApplicationControllerState provides the subset of controller state
 // required by the CAAS operator provisioner facade.
 type CAASApplicationControllerState interface {
 	ControllerConfig() (controller.Config, error)

caas/kubernetes/provider/bootstrap_test.go

@@ -77,6 +77,7 @@ func (s *bootstrapSuite) SetUpTest(c *gc.C) {
 	s.cfg = cfg
 
 	s.controllerCfg = coretesting.FakeControllerConfig()
+	s.controllerCfg["juju-db-snap-channel"] = controller.DefaultJujuDBSnapChannel
 	s.controllerCfg[controller.CAASImageRepo] = `
 {
     "serveraddress": "quay.io",

cloudconfig/podcfg/image.go

@@ -19,22 +19,32 @@ const (
 	JujudOCINamespace = "jujusolutions"
 	JujudOCIName      = "jujud-operator"
 	JujudbOCIName     = "juju-db"
-	JujudbVersion     = "4.4"
 )
 
 // GetControllerImagePath returns oci image path of jujud for a controller.
 func (cfg *ControllerPodConfig) GetControllerImagePath() (string, error) {
 	return GetJujuOCIImagePath(cfg.Controller.Config, cfg.JujuVersion, cfg.OfficialBuild)
 }
 
+func (cfg *ControllerPodConfig) dbVersion() (version.Number, error) {
+	snapChannel := cfg.Controller.Config.JujuDBSnapChannel()
+	vers := strings.Split(snapChannel, "/")[0] + ".0"
+	return version.Parse(vers)
+}
+
 // GetJujuDbOCIImagePath returns the juju-db oci image path.
 func (cfg *ControllerPodConfig) GetJujuDbOCIImagePath() (string, error) {
 	imageRepo := cfg.Controller.Config.CAASImageRepo().Repository
 	if imageRepo == "" {
 		imageRepo = JujudOCINamespace
 	}
 	path := fmt.Sprintf("%s/%s", imageRepo, JujudbOCIName)
-	return tagImagePath(path, JujudbVersion)
+	mongoVers, err := cfg.dbVersion()
+	if err != nil {
+		return "", errors.Annotatef(err, "cannot parse %q from controller config", controller.JujuDBSnapChannel)
+	}
+	tag := fmt.Sprintf("%d.%d", mongoVers.Major, mongoVers.Minor)
+	return tagImagePath(path, tag)
 }
 
 // IsJujuOCIImage returns true if the image path is for a Juju operator.

cloudconfig/podcfg/podcfg_test.go

@@ -57,6 +57,7 @@ func testPodLabels(c *gc.C, cfg *config.Config, jobs []model.MachineJob, expectT
 
 func (*podcfgSuite) TestOperatorImagesDefaultRepo(c *gc.C) {
 	cfg := testing.FakeControllerConfig()
+	cfg["juju-db-snap-channel"] = "4.4/stable"
 	podConfig, err := podcfg.NewBootstrapControllerPodConfig(
 		cfg,
 		"controller-1",
@@ -77,6 +78,7 @@ func (*podcfgSuite) TestOperatorImagesDefaultRepo(c *gc.C) {
 func (*podcfgSuite) TestOperatorImagesCustomRepo(c *gc.C) {
 	cfg := testing.FakeControllerConfig()
 	cfg["caas-image-repo"] = "path/to/my/repo"
+	cfg["juju-db-snap-channel"] = "4.4"
 	podConfig, err := podcfg.NewBootstrapControllerPodConfig(
 		cfg,
 		"controller-1",

cmd/juju/machine/add.go

@@ -111,7 +111,7 @@ Examples:
 	juju add-machine --constraints mem=8G
 
 	# Start a new machine within the "us-east-1a" availability zone.
-	juju add-machine --constraints zone=us-east-1a
+	juju add-machine --constraints zones=us-east-1a
 
 	# Start a new machine with at least 4 CPU cores and 16GB RAM, and request
 	# three storage volumes to be attached to it. Two are large capacity (1TB)

core/network/linklayer.go

@@ -29,13 +29,16 @@ const (
 
 	// BridgeDevice is used for OSI layer-2 bridge devices.
 	BridgeDevice LinkLayerDeviceType = "bridge"
+
+	// VXLANDevice is used for Virtual Extensible LAN devices.
+	VXLANDevice LinkLayerDeviceType = "vxlan"
 )
 
 // IsValidLinkLayerDeviceType returns whether the given value is a valid
 // link-layer network device type.
 func IsValidLinkLayerDeviceType(value string) bool {
 	switch LinkLayerDeviceType(value) {
-	case LoopbackDevice, EthernetDevice, VLAN8021QDevice, BondDevice, BridgeDevice:
+	case LoopbackDevice, EthernetDevice, VLAN8021QDevice, BondDevice, BridgeDevice, VXLANDevice:
 		return true
 	}
 	return false

core/network/linklayer_test.go

@@ -24,6 +24,7 @@ func (s *linkLayerSuite) TestIsValidLinkLayerDeviceTypeValid(c *gc.C) {
 		VLAN8021QDevice,
 		BondDevice,
 		BridgeDevice,
+		VXLANDevice,
 	}
 
 	for _, value := range validTypes {

core/network/source_netlink.go

@@ -62,6 +62,8 @@ func (n netlinkNIC) Type() LinkLayerDeviceType {
 		return VLAN8021QDevice
 	case "bond":
 		return BondDevice
+	case "vxlan":
+		return VXLANDevice
 	}
 
 	if n.nic.Attrs().Flags&net.FlagLoopback > 0 {

core/network/source_netlink_test.go

@@ -64,10 +64,16 @@ func (s *sourceNetlinkSuite) TestNetlinkNICType(c *gc.C) {
 	link := &stubLink{}
 	nic := &netlinkNIC{nic: link}
 
-	// If we have get value, return it.
+	// Known types.
+	link.linkType = "bridge"
+	c.Check(nic.Type(), gc.Equals, BridgeDevice)
+
 	link.linkType = "bond"
 	c.Check(nic.Type(), gc.Equals, BondDevice)
 
+	link.linkType = "vxlan"
+	c.Check(nic.Type(), gc.Equals, VXLANDevice)
+
 	// Infer loopback from flags.
 	link.linkType = ""
 	link.flags = net.FlagUp | net.FlagLoopback

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/iam v1.9.0
 	github.com/aws/smithy-go v1.8.0
 	github.com/bmizerany/pat v0.0.0-20160217103242-c068ca2f0aac
-	github.com/canonical/pebble v0.0.0-20211004014426-2fbdfdef40c7
+	github.com/canonical/pebble v0.0.0-20211017212823-71d8e376806f
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/coreos/go-systemd/v22 v22.0.0-20200316104309-cb8b64719ae3
 	github.com/dnaeon/go-vcr v1.1.0 // indirect

go.sum

@@ -123,8 +123,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bmizerany/pat v0.0.0-20160217103242-c068ca2f0aac h1:X5YRFJiteUM3rajABEYJSzw1KWgmp1ulPFKxpfLm0M4=
 github.com/bmizerany/pat v0.0.0-20160217103242-c068ca2f0aac/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
-github.com/canonical/pebble v0.0.0-20211004014426-2fbdfdef40c7 h1:+ZeDZM45AI/MSQF7W4DFL77SdTKtsc4Woelr0U/atX8=
-github.com/canonical/pebble v0.0.0-20211004014426-2fbdfdef40c7/go.mod h1:+0rQ57rhB9pciKKaE/QlwPL4R8mujv+24D81KGYRlV0=
+github.com/canonical/pebble v0.0.0-20211017212823-71d8e376806f h1:5SP7EqeBNxfAINGbYxIo5hHerEX1h7gw21Z2xbupx18=
+github.com/canonical/pebble v0.0.0-20211017212823-71d8e376806f/go.mod h1:+0rQ57rhB9pciKKaE/QlwPL4R8mujv+24D81KGYRlV0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

provider/openstack/provider.go

@@ -1418,11 +1418,12 @@ func (e *Environ) networksForInstance(
 		return nil, errors.Trace(err)
 	}
 
-	if !args.Constraints.HasSpaces() {
+	if len(args.SubnetsToZones) == 0 {
 		return networks, nil
 	}
 	if len(networks) == 0 {
-		return nil, errors.New("space constraints were supplied, but no Openstack network is configured")
+		return nil, errors.New(
+			"space constraints and/or bindings were supplied, but no Openstack network is configured")
 	}
 
 	// We know that we are operating in the single configured network.
@@ -1452,7 +1453,7 @@ func (e *Environ) networksForInstance(
 		subnetIDsForZone[i] = network.FilterInFanNetwork(subnetIDs)
 	}
 
-	/// For each list of subnet IDs that satisfy space and zone constraints,
+	// For each list of subnet IDs that satisfy space and zone constraints,
 	// choose a single one at random.
 	subnetIDForZone := make([]network.Id, len(subnetIDsForZone))
 	for i, subnetIDs := range subnetIDsForZone {

provider/openstack/provider_test.go

@@ -530,7 +530,7 @@ var handlerFunc = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request)
 	// This line is critical, the openstack provider will reject the message
 	// if you return 200 like a mere mortal.
 	w.WriteHeader(http.StatusMultipleChoices)
-	fmt.Fprint(w, `
+	_, _ = fmt.Fprint(w, `
 {
   "versions": {
     "values": [
@@ -920,11 +920,6 @@ func (s *providerUnitTests) TestNetworksForInstanceWithAZ(c *gc.C) {
 	siParams := environs.StartInstanceParams{
 		AvailabilityZone: "eu-west-az",
 		SubnetsToZones:   []map[network.Id][]string{{"subnet-foo": {"eu-west-az", "eu-east-az"}}},
-		Constraints: constraints.Value{
-			Spaces: &[]string{
-				"eu-west-az",
-			},
-		},
 	}
 
 	result, err := envWithNetworking(mockNetworking).networksForInstance(siParams, netCfg)

service/snap/snap_test.go

@@ -56,7 +56,7 @@ func (*validationSuite) TestValidateJujuDbSnap(c *gc.C) {
 	c.Check(err, jc.ErrorIsNil)
 
 	// via NewService
-	jujudbService, err := NewService("juju-db", "", common.Conf{Desc: "juju-db snap"}, Command, "/ath/to/config", "edge", "jailmode", []BackgroundService{}, []Installable{})
+	jujudbService, err := NewService("juju-db", "", common.Conf{Desc: "juju-db snap"}, Command, "/path/to/config", "edge", "jailmode", []BackgroundService{}, []Installable{})
 	c.Check(err, jc.ErrorIsNil)
 	c.Check(jujudbService.Validate(), jc.ErrorIsNil)
 
@@ -96,7 +96,7 @@ func (*snapSuite) TestConfigOverride(c *gc.C) {
 			"nofile": "64000",
 		},
 	}
-	svc, err := NewService("juju-db", "", conf, Command, "/ath/to/config", "latest", "strict", []BackgroundService{{
+	svc, err := NewService("juju-db", "", conf, Command, "/path/to/config", "latest", "strict", []BackgroundService{{
 		Name: "daemon",
 	}}, nil)
 	c.Assert(err, jc.ErrorIsNil)
@@ -134,14 +134,14 @@ func (*serviceSuite) TestInstallCommands(c *gc.C) {
 			EnableAtStartup: true,
 		},
 	}
-	service, err := NewService("juju-db", "juju-db", conf, Command, "/ath/to/config", "4.0/stable", "", backgroundServices, prerequisites)
+	service, err := NewService("juju-db", "juju-db", conf, Command, "/path/to/config", "4.4/stable", "", backgroundServices, prerequisites)
 	c.Assert(err, jc.ErrorIsNil)
 
 	commands, err := service.InstallCommands()
 	c.Assert(err, jc.ErrorIsNil)
 	c.Assert(commands, gc.DeepEquals, []string{
 		"snap install core",
-		"snap install --channel=4.0/stable juju-db",
+		"snap install --channel=4.4/stable juju-db",
 	})
 }
 
@@ -154,14 +154,14 @@ func (*serviceSuite) TestInstallCommandsWithConfinementPolicy(c *gc.C) {
 			EnableAtStartup: true,
 		},
 	}
-	service, err := NewService("juju-db", "juju-db", conf, Command, "/ath/to/config", "4.0/stable", "classic", backgroundServices, prerequisites)
+	service, err := NewService("juju-db", "juju-db", conf, Command, "/path/to/config", "4.4/stable", "classic", backgroundServices, prerequisites)
 	c.Assert(err, jc.ErrorIsNil)
 
 	commands, err := service.InstallCommands()
 	c.Assert(err, jc.ErrorIsNil)
 	c.Assert(commands, gc.DeepEquals, []string{
 		"snap install core",
-		"snap install --channel=4.0/stable --classic juju-db",
+		"snap install --channel=4.4/stable --classic juju-db",
 	})
 }
 
@@ -174,7 +174,7 @@ func (*serviceSuite) TestInstall(c *gc.C) {
 
 	runnable := NewMockRunnable(ctrl)
 	runnable.EXPECT().Execute("snap", []string{"install", "core"}).Return("", nil)
-	runnable.EXPECT().Execute("snap", []string{"install", "--channel=4.0/stable", "juju-db"}).Return("", nil)
+	runnable.EXPECT().Execute("snap", []string{"install", "--channel=4.4/stable", "juju-db"}).Return("", nil)
 
 	conf := common.Conf{}
 	prerequisites := []Installable{NewNamedApp("core")}
@@ -184,7 +184,7 @@ func (*serviceSuite) TestInstall(c *gc.C) {
 			EnableAtStartup: true,
 		},
 	}
-	service, err := NewService("juju-db", "juju-db", conf, Command, "/ath/to/config", "4.0/stable", "", backgroundServices, prerequisites)
+	service, err := NewService("juju-db", "juju-db", conf, Command, "/path/to/config", "4.4/stable", "", backgroundServices, prerequisites)
 	c.Assert(err, jc.ErrorIsNil)
 
 	s := &service
@@ -214,7 +214,7 @@ func (*serviceSuite) TestInstallWithRetry(c *gc.C) {
 	runnable := NewMockRunnable(ctrl)
 	runnable.EXPECT().Execute("snap", []string{"install", "core"}).Return("", errors.New("bad"))
 	runnable.EXPECT().Execute("snap", []string{"install", "core"}).Return("", nil)
-	runnable.EXPECT().Execute("snap", []string{"install", "--channel=4.0/stable", "juju-db"}).Return("", nil)
+	runnable.EXPECT().Execute("snap", []string{"install", "--channel=4.4/stable", "juju-db"}).Return("", nil)
 
 	conf := common.Conf{}
 	prerequisites := []Installable{NewNamedApp("core")}
@@ -224,7 +224,7 @@ func (*serviceSuite) TestInstallWithRetry(c *gc.C) {
 			EnableAtStartup: true,
 		},
 	}
-	service, err := NewService("juju-db", "juju-db", conf, Command, "/ath/to/config", "4.0/stable", "", backgroundServices, prerequisites)
+	service, err := NewService("juju-db", "juju-db", conf, Command, "/path/to/config", "4.4/stable", "", backgroundServices, prerequisites)
 	c.Assert(err, jc.ErrorIsNil)
 
 	s := &service

snap/local/wrappers/fetch-oci

@@ -32,7 +32,7 @@ echo "Wait for microk8s to be ready if needed."
 microk8s.status --wait-ready --timeout 30 2>&1
 juju_version=\$(/snap/bin/juju version | rev | cut -d- -f3- | rev)
 oci_image="docker.io/jujusolutions/jujud-operator:\$juju_version"
-mongo_image="docker.io/jujusolutions/juju-db:4.0"
+mongo_image="docker.io/jujusolutions/juju-db:4.4"
 
 echo "Going to cache images: \$oci_image and \$mongo_image."
 echo "Pulling: \$oci_image."

state/application.go

@@ -2200,16 +2200,6 @@ type applicationAddUnitOpsArgs struct {
 	unitName   *string
 }
 
-// addApplicationUnitOps is just like addUnitOps but explicitly takes a
-// constraints value (this is used at application creation time).
-func (a *Application) addApplicationUnitOps(args applicationAddUnitOpsArgs) (string, []txn.Op, error) {
-	result, ops, err := a.addUnitOpsWithCons(args)
-	if err == nil {
-		ops = append(ops, a.incUnitCountOp(nil))
-	}
-	return result, ops, err
-}
-
 // addUnitOpsWithCons is a helper method for returning addUnitOps.
 func (a *Application) addUnitOpsWithCons(args applicationAddUnitOpsArgs) (string, []txn.Op, error) {
 	if a.doc.Subordinate && args.principalName == "" {

state/state.go

@@ -1187,6 +1187,7 @@ func (st *State) AddApplication(args AddApplicationArgs) (_ *Application, err er
 		Channel:       string(args.Channel),
 		RelationCount: len(peers),
 		Life:          Alive,
+		UnitCount:     args.NumUnits,
 
 		// CAAS
 		DesiredScale: scale,
@@ -1292,7 +1293,7 @@ func (st *State) AddApplication(args AddApplicationArgs) (_ *Application, err er
 
 		// Collect unit-adding operations.
 		for x := 0; x < args.NumUnits; x++ {
-			unitName, unitOps, err := app.addApplicationUnitOps(applicationAddUnitOpsArgs{
+			unitName, unitOps, err := app.addUnitOpsWithCons(applicationAddUnitOpsArgs{
 				cons:          args.Constraints,
 				storageCons:   args.Storage,
 				attachStorage: args.AttachStorage,