Merge pull request #9019 from babbageclunk/fix-lease-store-race

#9019

## Description of change

This fixes a data race caused by the lease manager calling store methods
asynchronously now.

Example race test failure: http://10.125.0.203:8080/job/RunUnittests-race-amd64/584/testReport/github/com_juju_juju_worker_uniter_runner_context/TestPackage/

## QA steps

* Run failing tests under the race checker.

state/lease/store.go

@@ -5,6 +5,7 @@ package lease
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/juju/errors"
@@ -44,6 +45,7 @@ func NewStore(config StoreConfig) (lease.Store, error) {
 
 // store implements the lease.Store interface.
 type store struct {
+	mu sync.Mutex
 
 	// config holds resources and configuration necessary to store leases.
 	config StoreConfig
@@ -60,6 +62,8 @@ type store struct {
 
 // Leases is part of the lease.Store interface.
 func (store *store) Leases() map[lease.Key]lease.Info {
+	store.mu.Lock()
+	defer store.mu.Unlock()
 	localTime := store.config.LocalClock.Now()
 	leases := make(map[lease.Key]lease.Info)
 	for name, entry := range store.entries {
@@ -102,6 +106,9 @@ func (store *store) request(name string, request lease.Request, getOps opsFunc,
 		return errors.Annotatef(err, "invalid request")
 	}
 
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
 	// Close over cacheEntry to record in case of success.
 	var cacheEntry entry
 	err := store.config.Mongo.RunTransaction(func(attempt int) ([]txn.Op, error) {
@@ -147,6 +154,9 @@ func (store *store) ExpireLease(key lease.Key) error {
 		return errors.Annotatef(err, "invalid name")
 	}
 
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
 	// No cache updates needed, only deletes; no closure here.
 	err := store.config.Mongo.RunTransaction(func(attempt int) ([]txn.Op, error) {
 		store.logger.Tracef("expiring lease %q (attempt %d)", name, attempt)
@@ -180,6 +190,9 @@ func (store *store) ExpireLease(key lease.Key) error {
 
 // Refresh is part of the Store interface.
 func (store *store) Refresh() error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
 	return store.refresh(true)
 }