From f66954a30a56f3fcacc9e4a6db56ff2088419555 Mon Sep 17 00:00:00 2001 From: Flavio Castelli Date: Fri, 28 Jun 2024 14:09:49 +0200 Subject: [PATCH 1/2] fix: address issues of sqlite backend Using sqlite to save checkpoints failed at runtime because: * Internal Mutex was a nil pointer * The help messages explaining how to create the sqlite3 database had the wrong CREATE statement * The implementation of the `LockBackend` interface had the `Fetch` function return a wrong object (a `dynamoCheckpoint` instead of a `sqliteCheckpoint`) Signed-off-by: Flavio Castelli --- cmd/sunlight/main.go | 2 +- internal/ctlog/sqlite.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/sunlight/main.go b/cmd/sunlight/main.go index 63d7750..0e6a49d 100644 --- a/cmd/sunlight/main.go +++ b/cmd/sunlight/main.go @@ -80,7 +80,7 @@ type Config struct { // The database must already exist to protect against accidental // misconfiguration. Create the table with: // - // $ sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, checkpoint TEXT)" + // $ sqlite3 checkpoints.db "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body TEXT)" // Checkpoints string diff --git a/internal/ctlog/sqlite.go b/internal/ctlog/sqlite.go index 9c0abfb..2764dc2 100644 --- a/internal/ctlog/sqlite.go +++ b/internal/ctlog/sqlite.go @@ -34,7 +34,7 @@ func NewSQLiteBackend(ctx context.Context, path string, l *slog.Logger) (*SQLite conn, err := sqlite.OpenConn(path, sqlite.OpenFlagsDefault & ^sqlite.SQLITE_OPEN_CREATE) if err != nil { - return nil, fmt.Errorf(`failed to open SQLite lock database (hint: to avoid misconfiguration, the lock database must be created manually with "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, checkpoint TEXT)"): %w`, err) + return nil, fmt.Errorf(`failed to open SQLite lock database (hint: to avoid misconfiguration, the lock database must be created manually with "CREATE TABLE checkpoints (logID BLOB PRIMARY KEY, body TEXT)"): %w`, err) } if err := sqlitex.ExecTransient(conn, "PRAGMA synchronous = FULL", nil); err != nil { conn.Close() @@ -49,6 +49,7 @@ func NewSQLiteBackend(ctx context.Context, path string, l *slog.Logger) (*SQLite conn: conn, duration: duration, log: l, + mu: &sync.Mutex{}, }, nil } @@ -78,7 +79,7 @@ func (b *SQLiteBackend) Fetch(ctx context.Context, logID [sha256.Size]byte) (Loc if body == nil { return nil, errors.New("checkpoint not found") } - return &dynamoDBCheckpoint{logID: logID, body: body}, nil + return &sqliteCheckpoint{logID: logID, body: body}, nil } func (b *SQLiteBackend) Replace(ctx context.Context, old LockedCheckpoint, new []byte) (LockedCheckpoint, error) { From fec68c1eade796b9b70e3ee432a6cf06d930d6af Mon Sep 17 00:00:00 2001 From: Flavio Castelli Date: Fri, 28 Jun 2024 14:46:24 +0200 Subject: [PATCH 2/2] feat: introduce local backend Prior to this commit, sunlight required access to a S3-compatible server to send backend data. Now there's the possibility to store this data on the local filesystem. This is not meant to be used in production, but it hugely simplifies testing and local development. Signed-off-by: Flavio Castelli --- cmd/sunlight/main.go | 30 +++++++++++++--- internal/ctlog/local.go | 80 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 internal/ctlog/local.go diff --git a/cmd/sunlight/main.go b/cmd/sunlight/main.go index 0e6a49d..fc40f99 100644 --- a/cmd/sunlight/main.go +++ b/cmd/sunlight/main.go @@ -181,6 +181,14 @@ type LogConfig struct { // going to be treated like a directory in many tools using S3. S3KeyPrefix string + // LocalBackend is the path to the directory where backend data is going to + // be saved. + // + // Not meant to be used in production, only for testing and development purposes. + // + // Cannot be used at the same time as the S3 bucket. + LocalBackend string + // NotAfterStart is the start of the validity range for certificates // accepted by this log instance, as and RFC 3339 date. NotAfterStart string @@ -306,10 +314,24 @@ func main() { slog.String("log", lc.ShortName), })) - b, err := ctlog.NewS3Backend(ctx, lc.S3Region, lc.S3Bucket, lc.S3Endpoint, lc.S3KeyPrefix, logger) - if err != nil { - logger.Error("failed to create backend", "err", err) - os.Exit(1) + var b ctlog.Backend + if lc.LocalBackend != "" { + if lc.S3Bucket != "" || lc.S3Region != "" || lc.S3Endpoint != "" || lc.S3KeyPrefix != "" { + logger.Error("local backend cannot be used with S3") + os.Exit(1) + } + + b, err = ctlog.NewLocalBackend(lc.LocalBackend) + if err != nil { + logger.Error("failed to create backend", "err", err) + os.Exit(1) + } + } else { + b, err = ctlog.NewS3Backend(ctx, lc.S3Region, lc.S3Bucket, lc.S3Endpoint, lc.S3KeyPrefix, logger) + if err != nil { + logger.Error("failed to create backend", "err", err) + os.Exit(1) + } } r := x509util.NewPEMCertPool() diff --git a/internal/ctlog/local.go b/internal/ctlog/local.go new file mode 100644 index 0000000..a02c36b --- /dev/null +++ b/internal/ctlog/local.go @@ -0,0 +1,80 @@ +package ctlog + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "path" + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +// LocalFilesystemBackend is a backend that stores key-value pairs in the +// local filesystem. +// The keys are base64-encoded and the values are stored in files named after +// the base64-encoded key. +// +// This is not meant to be used in production, but rather for testing and +// development purposes. +type LocalFilesystemBackend struct { + mu *sync.RWMutex + rootPath string +} + +func NewLocalBackend(path string) (*LocalFilesystemBackend, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(path, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory %s: %w", path, err) + } + } else { + return nil, fmt.Errorf("failed to check if directory %s exists: %w", path, err) + } + } + + return &LocalFilesystemBackend{ + mu: &sync.RWMutex{}, + rootPath: path, + }, nil +} + +// Upload saves the data associated to the key in the local filesystem. +// Note well: upload options are not handled +func (b *LocalFilesystemBackend) Upload(ctx context.Context, key string, data []byte, opts *UploadOptions) error { + b.mu.Lock() + defer b.mu.Unlock() + filename := keyToFilename(b.rootPath, key) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("failed to save key %s to file %s: %w", key, filename, err) + } + defer f.Close() + if _, err := f.Write(data); err != nil { + return fmt.Errorf("failed to write contents of key %s to file %s: %w", key, filename, err) + } + return nil +} + +func (b *LocalFilesystemBackend) Fetch(ctx context.Context, key string) ([]byte, error) { + b.mu.RLock() + defer b.mu.RUnlock() + + filename := keyToFilename(b.rootPath, key) + + data, err := os.ReadFile(filename) + if err != nil { + return []byte{}, fmt.Errorf("failed to read contents of file %s associated to key %s: %w", filename, key, err) + } + + return data, nil +} + +func (b *LocalFilesystemBackend) Metrics() []prometheus.Collector { + return []prometheus.Collector{} +} + +func keyToFilename(rootPath, key string) string { + return path.Join(rootPath, base64.StdEncoding.EncodeToString([]byte(key))) +}