[JUJU-210] DQLITE scaffolding work

Makefile

@@ -8,7 +8,12 @@ GOOS=$(shell go env GOOS)
 GOARCH=$(shell go env GOARCH)
 GOHOSTOS=$(shell go env GOHOSTOS)
 GOHOSTARCH=$(shell go env GOHOSTARCH)
-export CGO_ENABLED=0
+
+CGO_ENABLED=1
+CGO_CFLAGS=-I$(GOPATH)/deps/raft/include/ -I$(GOPATH)/deps/dqlite/include/
+CGO_LDFLAGS=-L$(GOPATH)/deps/raft/.libs -L$(GOPATH)/deps/dqlite/.libs/
+LD_LIBRARY_PATH=$(GOPATH)/deps/raft/.libs/:$(GOPATH)/deps/dqlite/.libs/
+CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
 
 BUILD_DIR ?= $(PROJECT_DIR)/_build
 BIN_DIR = ${BUILD_DIR}/${GOOS}_${GOARCH}/bin
@@ -59,6 +64,7 @@ GIT_TREE_STATE = $(if $(shell git -C $(PROJECT_DIR) rev-parse --is-inside-work-t
 # Build tags passed to go install/build.
 # Example: BUILD_TAGS="minimal provider_kubernetes"
 BUILD_TAGS ?=
+REQUIRED_BUILD_TAGS = libsqlite3
 
 # Build number passed in must be a monotonic int representing
 # the build.
@@ -80,7 +86,7 @@ ifeq ($(shell echo "${GOARCH}" | sed -E 's/.*(ppc64le|ppc64).*/golang/'), golang
 else
 	COMPILE_FLAGS =
 endif
-    LINK_FLAGS = -ldflags "-s -w -extldflags '-static' -X $(PROJECT)/version.GitCommit=$(GIT_COMMIT) -X $(PROJECT)/version.GitTreeState=$(GIT_TREE_STATE) -X $(PROJECT)/version.build=$(JUJU_BUILD_NUMBER)"
+    LINK_FLAGS = -ldflags "-s -w  -X $(PROJECT)/version.GitCommit=$(GIT_COMMIT) -X $(PROJECT)/version.GitTreeState=$(GIT_TREE_STATE) -X $(PROJECT)/version.build=$(JUJU_BUILD_NUMBER)"
 endif
 
 define DEPENDENCIES
@@ -123,8 +129,8 @@ run-tests:
 ## run-tests: Run the unit tests
 	$(eval TMP := $(shell mktemp -d $${TMPDIR:-/tmp}/jj-XXX))
 	$(eval TEST_PACKAGES := $(shell go list $(PROJECT)/... | grep -v $(PROJECT)$$ | grep -v $(PROJECT)/vendor/ | grep -v $(PROJECT)/acceptancetests/ | grep -v $(PROJECT)/generate/ | grep -v mocks))
-	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v'
-	@TMPDIR=$(TMP) go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v
+	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v'
+	@TMPDIR=$(TMP) CGO_LDFLAGS_ALLOW=$(CGO_LDFLAGS_ALLOW) go test -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v
 	@rm -r $(TMP)
 
 install: rebuild-schema go-install
@@ -136,14 +142,13 @@ clean:
 
 go-install:
 ## go-install: Install Juju binaries without updating dependencies
-	@echo 'go install -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $$MAIN_PACKAGES'
-	@go install -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $(strip $(MAIN_PACKAGES))
+	@echo 'go install -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $$MAIN_PACKAGES'
+	@CGO_LDFLAGS_ALLOW=$(CGO_LDFLAGS_ALLOW) go install -mod=$(JUJU_GOMOD_MODE) -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $(strip $(MAIN_PACKAGES))
 
 go-build:
 ## go-build: Build Juju binaries without updating dependencies
-	@mkdir -p ${BIN_DIR}
-	@echo 'go build -mod=$(JUJU_GOMOD_MODE) -o ${BIN_DIR} -tags "$(BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $$MAIN_PACKAGES'
-	@go build -mod=$(JUJU_GOMOD_MODE) -o ${BIN_DIR} -tags "$(BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $(strip $(MAIN_PACKAGES))
+	@mkdir -p ${BIN_DIR} #@echo 'CGO_LDFLAGS_ALLOW=$(CGO_LDFLAGS_ALLOW) go build -mod=$(JUJU_GOMOD_MODE) -o ${BIN_DIR} -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $$MAIN_PACKAGES'
+	CC="$(CC)" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_LDFLAGS_ALLOW=$(CGO_LDFLAGS_ALLOW) CGO_ENABLED=1 go build -mod=$(JUJU_GOMOD_MODE) -o ${BIN_DIR} -tags "$(BUILD_TAGS) $(REQUIRED_BUILD_TAGS)" $(COMPILE_FLAGS) $(LINK_FLAGS) -v $(strip $(MAIN_PACKAGES))
 
 vendor-dependencies:
 ## vendor-dependencies: updates vendored dependencies
@@ -281,3 +286,40 @@ static-analysis:
 .PHONY: clean format simplify test run-tests
 .PHONY: install-dependencies
 .PHONY: check-deps
+
+
+RAFT_PATH=$(GOPATH)/deps/raft
+DQLITE_PATH=$(GOPATH)/deps/dqlite
+
+.PHONY: deps
+deps:
+	@if [ ! -e "$(RAFT_PATH)" ]; then \
+		git clone --depth=1 "https://github.com/canonical/raft" "$(RAFT_PATH)"; \
+	elif [ -e "$(RAFT_PATH)/.git" ]; then \
+		cd "$(RAFT_PATH)"; git pull; \
+	fi
+
+	cd "$(RAFT_PATH)" && \
+		autoreconf -i && \
+		./configure && \
+		make
+
+	# dqlite
+	@if [ ! -e "$(DQLITE_PATH)" ]; then \
+		git clone --depth=1 "https://github.com/canonical/dqlite" "$(DQLITE_PATH)"; \
+	elif [ -e "$(DQLITE_PATH)/.git" ]; then \
+		cd "$(DQLITE_PATH)"; git pull; \
+	fi
+
+	cd "$(DQLITE_PATH)" && \
+		autoreconf -i && \
+		PKG_CONFIG_PATH="$(RAFT_PATH)" ./configure && \
+		make CFLAGS="-I$(RAFT_PATH)/include/" LDFLAGS="-L$(RAFT_PATH)/.libs/"
+
+	# environment
+	@echo ""
+	@echo "Please set the following in your environment (possibly ~/.bashrc)"
+	@echo "export CGO_CFLAGS=\"-I$(RAFT_PATH)/include/ -I$(DQLITE_PATH)/include/\""
+	@echo "export CGO_LDFLAGS=\"-L$(RAFT_PATH)/.libs -L$(DQLITE_PATH)/.libs/\""
+	@echo "export LD_LIBRARY_PATH=\"$(RAFT_PATH)/.libs/:$(DQLITE_PATH)/.libs/\""
+	@echo "export CGO_LDFLAGS_ALLOW=\"(-Wl,-wrap,pthread_create)|(-Wl,-z,now)\""

apiserver/apiserver.go

@@ -5,6 +5,7 @@ package apiserver
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"io"
 	"net/http"
@@ -58,6 +59,11 @@ var logger = loggo.GetLogger("juju.apiserver")
 
 var defaultHTTPMethods = []string{"GET", "POST", "HEAD", "PUT", "DELETE", "OPTIONS"}
 
+// SQLDBGetter provides access to model-scoped SQL databases.
+type SQLDBGetter interface {
+	GetDB(modelUUID string) (*sql.DB, error)
+}
+
 // Server holds the server side of the API.
 type Server struct {
 	tomb      tomb.Tomb
@@ -198,6 +204,9 @@ type ServerConfig struct {
 	// RaftOpQueue is used by the API to apply operations to the raft
 	// instance.
 	RaftOpQueue Queue
+
+	// SQLDBGetter provides access to model-scoped SQL databases.
+	SQLDBGetter SQLDBGetter
 }
 
 // Validate validates the API server configuration.
@@ -249,6 +258,9 @@ func (c ServerConfig) Validate() error {
 	if c.RaftOpQueue == nil {
 		return errors.NotValidf("missing RaftOpQueue")
 	}
+	if c.SQLDBGetter == nil {
+		return errors.NotValidf("missing SQLDBGetter")
+	}
 	return nil
 }
 
@@ -298,6 +310,7 @@ func newServer(cfg ServerConfig) (_ *Server, err error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
+
 	srv := &Server{
 		clock:                         cfg.Clock,
 		pingClock:                     cfg.pingClock(),
@@ -324,6 +337,7 @@ func newServer(cfg ServerConfig) (_ *Server, err error) {
 			clock:                 cfg.Clock,
 			dbLoggerBufferSize:    cfg.LogSinkConfig.DBLoggerBufferSize,
 			dbLoggerFlushInterval: cfg.LogSinkConfig.DBLoggerFlushInterval,
+			sqlDBGetter:           cfg.SQLDBGetter,
 		},
 		metricsCollector:    cfg.MetricsCollector,
 		execEmbeddedCommand: cfg.ExecEmbeddedCommand,

apiserver/logsink.go

@@ -4,6 +4,7 @@
 package apiserver
 
 import (
+	"database/sql"
 	"io"
 	"net/http"
 	"strings"
@@ -50,6 +51,7 @@ type dbloggers struct {
 	clock                 clock.Clock
 	dbLoggerBufferSize    int
 	dbLoggerFlushInterval time.Duration
+	sqlDBGetter           SQLDBGetter
 	mu                    sync.Mutex
 	loggers               map[*state.State]*bufferedDbLogger
 }
@@ -63,9 +65,13 @@ func (d *dbloggers) get(st *state.State) recordLogger {
 	if d.loggers == nil {
 		d.loggers = make(map[*state.State]*bufferedDbLogger)
 	}
+
 	dbl := state.NewDbLogger(st)
 	l := &bufferedDbLogger{dbl, logdb.NewBufferedLogger(
-		dbl,
+		logdb.NewTeeLogger(
+			dbl,
+			newSQLLogger(d.sqlDBGetter),
+		),
 		d.dbLoggerBufferSize,
 		d.dbLoggerFlushInterval,
 		d.clock,
@@ -218,3 +224,50 @@ func logToFile(writer io.Writer, prefix string, m params.LogRecord) error {
 	}, " ") + "\n"))
 	return err
 }
+
+// sqlLogger inserts log entries to an sql.DB instance.
+type sqlLogger struct {
+	db    *sql.DB
+	dbErr error
+}
+
+func newSQLLogger(dbGetter SQLDBGetter) *sqlLogger {
+	db, err := dbGetter.GetDB("logs")
+	return &sqlLogger{
+		db:    db,
+		dbErr: err,
+	}
+}
+
+const (
+	sqlInsertLogEntry = "INSERT INTO logs (ts, entity, version, module, location, level, message, labels) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
+)
+
+func (l *sqlLogger) Log(records []state.LogRecord) error {
+	if l.dbErr != nil {
+		return l.dbErr
+	}
+
+	tx, err := l.db.Begin()
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	for _, r := range records {
+		if _, err = tx.Exec(sqlInsertLogEntry,
+			r.Time,
+			r.Entity,
+			r.Version.String(),
+			r.Module,
+			r.Location,
+			r.Level,
+			r.Message,
+			strings.Join(r.Labels, ","),
+		); err != nil {
+			_ = tx.Rollback()
+			return errors.Trace(err)
+		}
+	}
+
+	return errors.Trace(tx.Commit())
+}

cmd/jujud/agent/machine/manifolds.go

@@ -52,6 +52,7 @@ import (
 	lxdbroker "github.com/juju/juju/worker/containerbroker"
 	"github.com/juju/juju/worker/controllerport"
 	"github.com/juju/juju/worker/credentialvalidator"
+	"github.com/juju/juju/worker/dbaccessor"
 	"github.com/juju/juju/worker/deployer"
 	"github.com/juju/juju/worker/diskmanager"
 	"github.com/juju/juju/worker/externalcontrollerupdater"
@@ -714,6 +715,8 @@ func commonManifolds(config ManifoldsConfig) dependency.Manifolds {
 			// can't shutdown properly.
 			RaftTransportName: raftTransportName,
 
+			DBAccessorName: dbAccessorName,
+
 			PrometheusRegisterer:              config.PrometheusRegisterer,
 			RegisterIntrospectionHTTPHandlers: config.RegisterIntrospectionHTTPHandlers,
 			Hub:                               config.CentralHub,
@@ -744,6 +747,13 @@ func commonManifolds(config ManifoldsConfig) dependency.Manifolds {
 			NewWorker:            peergrouper.New,
 		})),
 
+		dbAccessorName: ifController(dbaccessor.Manifold(dbaccessor.ManifoldConfig{
+			AgentName:      agentName,
+			CentralHubName: centralHubName,
+			Clock:          config.Clock,
+			Logger:         loggo.GetLogger("juju.worker.dbaccessor"),
+		})),
+
 		restoreWatcherName: restorewatcher.Manifold(restorewatcher.ManifoldConfig{
 			StateName: stateName,
 			NewWorker: restorewatcher.NewWorker,
@@ -1152,6 +1162,7 @@ const (
 	modelWorkerManagerName        = "model-worker-manager"
 	multiwatcherName              = "multiwatcher"
 	peergrouperName               = "peer-grouper"
+	dbAccessorName                = "db-accessor"
 	restoreWatcherName            = "restore-watcher"
 	certificateUpdaterName        = "certificate-updater"
 	auditConfigUpdaterName        = "audit-config-updater"

go.mod

@@ -21,9 +21,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/iam v1.9.0
 	github.com/aws/smithy-go v1.8.0
 	github.com/bmizerany/pat v0.0.0-20160217103242-c068ca2f0aac
+	github.com/canonical/go-dqlite v1.10.1
 	github.com/canonical/pebble v0.0.0-20211124224737-13a5e875b6a1
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
-	github.com/coreos/go-systemd/v22 v22.0.0-20200316104309-cb8b64719ae3
+	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/dnaeon/go-vcr v1.1.0 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/dustin/go-humanize v1.0.0
@@ -32,7 +33,6 @@ require (
 	github.com/go-logr/logr v0.2.0
 	github.com/go-macaroon-bakery/macaroon-bakery/v3 v3.0.0-20210309064400-d73aa8f92aa2
 	github.com/golang/mock v1.5.0
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.0.0
 	github.com/google/uuid v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.4.1
@@ -43,7 +43,6 @@ require (
 	github.com/hashicorp/go-hclog v0.9.1
 	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5
-	github.com/hashicorp/go-uuid v1.0.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/raft v1.3.2-0.20210825230038-1a621031eb2b
 	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
@@ -95,6 +94,7 @@ require (
 	github.com/kr/pretty v0.2.1
 	github.com/lxc/lxd v0.0.0-20210607205159-a7c206b5233d
 	github.com/mattn/go-isatty v0.0.14
+	github.com/mattn/go-sqlite3 v1.14.7
 	github.com/mitchellh/go-linereader v0.0.0-20190213213312-1b945b3263eb
 	github.com/onsi/ginkgo v1.14.2 // indirect
 	github.com/onsi/gomega v1.10.4 // indirect
@@ -105,26 +105,20 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rogpeppe/fastuuid v1.2.0 // indirect
 	github.com/satori/go.uuid v1.2.0
-	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vmware/govmomi v0.21.1-0.20191008161538-40aebf13ba45
-	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
-	golang.org/x/mod v0.4.0 // indirect
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
-	google.golang.org/api v0.29.0
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/genproto v0.0.0-20200726014623-da3ae01ef02d // indirect
+	golang.org/x/tools v0.1.2
+	google.golang.org/api v0.44.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/httprequest.v1 v1.2.1
-	gopkg.in/ini.v1 v1.51.0
+	gopkg.in/ini.v1 v1.62.0
 	gopkg.in/juju/environschema.v1 v1.0.1-0.20201027142642-c89a4490670a
 	gopkg.in/macaroon.v2 v2.1.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0