Skip to content

Commit

Permalink
physical/mysql: cleanup and documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@armon
armon committed Jun 18, 2015
1 parent 748c850 commit 46ba8d1
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 164 deletions.
161 changes: 66 additions & 95 deletions physical/mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package physical

import (
"database/sql"
"errors"
"fmt"
"sort"
"strings"
Expand All @@ -12,101 +11,96 @@ import (
_ "github.com/go-sql-driver/mysql"
)

var (
MySQLPrepareStmtFailure = errors.New("failed to prepare statement")
MySQLExecuteStmtFailure = errors.New("failed to execute statement")
)

// MySQLBackend is a physical backend that stores data
// within MySQL database.
type MySQLBackend struct {
table string
database string
dbTable string
client *sql.DB
statements map[string]*sql.Stmt
}

// newMySQLBackend constructs a MySQL backend using the given API client and
// server address and credential for accessing mysql database.
func newMySQLBackend(conf map[string]string) (Backend, error) {
// Get the MySQL credentials to perform read/write operations.
username, ok := conf["username"]
if !ok || username == "" {
return nil, fmt.Errorf("missing username")
}
password, ok := conf["password"]
if !ok || username == "" {
return nil, fmt.Errorf("missing password")
}

// Get or set MySQL server address. Defaults to localhost and default port(3306)
address, ok := conf["address"]
if !ok {
address = "127.0.0.1:3306"
}

// Get the MySQL credentials to perform read/write operations.
username, ok := conf["username"]
password, ok := conf["password"]

// Get the MySQL database and table details.
database, ok := conf["database"]
if !ok {
return nil, fmt.Errorf("database name is missing in the configuration")
database = "vault"
}
table, ok := conf["table"]
if !ok {
return nil, fmt.Errorf("table name is missing in the configuration")
table = "vault"
}
dbTable := database + "." + table

// Create MySQL handle for the database.
dsn := username + ":" + password + "@tcp(" + address + ")/" + database
dsn := username + ":" + password + "@tcp(" + address + ")/"
db, err := sql.Open("mysql", dsn)
if err != nil {
return nil, fmt.Errorf("failed to open handler with database")
}

// Create the required table if it doesn't exists.
create_query := "CREATE TABLE IF NOT EXISTS " + database + "." + table + " (vault_key varchar(512), vault_value mediumblob, PRIMARY KEY (vault_key))"
create_stmt, err := db.Prepare(create_query)
if err != nil {
return nil, MySQLPrepareStmtFailure
return nil, fmt.Errorf("failed to connect to mysql: %v", err)
}
defer create_stmt.Close()

_, err = create_stmt.Exec()
if err != nil {
return nil, MySQLExecuteStmtFailure
// Create the required database if it doesn't exists.
if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS " + database); err != nil {
return nil, fmt.Errorf("failed to create mysql database: %v", err)
}

// Map of query type as key to prepared statement.
statements := make(map[string]*sql.Stmt)

// Prepare statement for put query.
insert_query := "INSERT INTO " + database + "." + table + " VALUES( ?, ? ) ON DUPLICATE KEY UPDATE vault_value=VALUES(vault_value)"
insert_stmt, err := db.Prepare(insert_query)
if err != nil {
return nil, MySQLPrepareStmtFailure
}
statements["put"] = insert_stmt

// Prepare statement for select query.
select_query := "SELECT vault_value FROM " + database + "." + table + " WHERE vault_key = ?"
select_stmt, err := db.Prepare(select_query)
if err != nil {
return nil, MySQLPrepareStmtFailure
}
statements["get"] = select_stmt

// Prepare statement for delete query.
delete_query := "DELETE FROM " + database + "." + table + " WHERE vault_key = ?"
delete_stmt, err := db.Prepare(delete_query)
if err != nil {
return nil, MySQLPrepareStmtFailure
// Create the required table if it doesn't exists.
create_query := "CREATE TABLE IF NOT EXISTS " + dbTable +
" (vault_key varchar(512), vault_value mediumblob, PRIMARY KEY (vault_key))"
if _, err := db.Exec(create_query); err != nil {
return nil, fmt.Errorf("failed to create mysql table: %v", err)
}
statements["delete"] = delete_stmt

// Setup the backend.
m := &MySQLBackend{
dbTable: dbTable,
client: db,
table: table,
database: database,
statements: statements,
statements: make(map[string]*sql.Stmt),
}

// Prepare all the statements required
statements := map[string]string{
"put": "INSERT INTO " + dbTable +
" VALUES( ?, ? ) ON DUPLICATE KEY UPDATE vault_value=VALUES(vault_value)",
"get": "SELECT vault_value FROM " + dbTable + " WHERE vault_key = ?",
"delete": "DELETE FROM " + dbTable + " WHERE vault_key = ?",
"list": "SELECT vault_key FROM " + dbTable + " WHERE vault_key LIKE ?",
}
for name, query := range statements {
if err := m.prepare(name, query); err != nil {
return nil, err
}
}
return m, nil
}

// prepare is a helper to prepare a query for future execution
func (m *MySQLBackend) prepare(name, query string) error {
stmt, err := m.client.Prepare(query)
if err != nil {
return fmt.Errorf("failed to prepare '%s': %v", name, err)
}
m.statements[name] = stmt
return nil
}

// Put is used to insert or update an entry.
func (m *MySQLBackend) Put(entry *Entry) error {
defer metrics.MeasureSince([]string{"mysql", "put"}, time.Now())
Expand All @@ -115,7 +109,6 @@ func (m *MySQLBackend) Put(entry *Entry) error {
if err != nil {
return err
}

return nil
}

Expand All @@ -124,22 +117,18 @@ func (m *MySQLBackend) Get(key string) (*Entry, error) {
defer metrics.MeasureSince([]string{"mysql", "get"}, time.Now())

var result []byte

err := m.statements["get"].QueryRow(key).Scan(&result)
if err != nil {
if err == sql.ErrNoRows {
return nil, nil
}

// Handle a non-existing value
if result == nil {
return nil, nil
if err != nil {
return nil, err
}

ent := &Entry{
Key: key,
Value: result,
}

return ent, nil
}

Expand All @@ -151,7 +140,6 @@ func (m *MySQLBackend) Delete(key string) error {
if err != nil {
return err
}

return nil
}

Expand All @@ -160,45 +148,28 @@ func (m *MySQLBackend) Delete(key string) error {
func (m *MySQLBackend) List(prefix string) ([]string, error) {
defer metrics.MeasureSince([]string{"mysql", "list"}, time.Now())

// Query to get all keys matching a prefix.
list_query := "SELECT vault_key FROM " + m.database + "." + m.table + " WHERE vault_key LIKE '" + prefix + "%'"
rows, err := m.client.Query(list_query)
if err != nil {
return nil, MySQLExecuteStmtFailure
}

columns, err := rows.Columns()
if err != nil {
return nil, fmt.Errorf("failed to get columns")
}

values := make([]sql.RawBytes, len(columns))
// Add the % wildcard to the prefix to do the prefix search
likePrefix := prefix + "%"
rows, err := m.statements["list"].Query(likePrefix)

scanArgs := make([]interface{}, len(values))
for i := range values {
scanArgs[i] = &values[i]
}

keys := []string{}
var keys []string
for rows.Next() {
err = rows.Scan(scanArgs...)
var key string
err = rows.Scan(&key)
if err != nil {
return nil, fmt.Errorf("failed to scan rows")
return nil, fmt.Errorf("failed to scan rows: %v", err)
}

for _, key := range values {
key := strings.TrimPrefix(string(key), prefix)
if i := strings.Index(string(key), "/"); i == -1 {
// Add objects only from the current 'folder'
keys = append(keys, string(key))
} else if i != -1 {
// Add truncated 'folder' paths
keys = appendIfMissing(keys, string(key[:i+1]))
}
key = strings.TrimPrefix(key, prefix)
if i := strings.Index(key, "/"); i == -1 {
// Add objects only from the current 'folder'
keys = append(keys, key)
} else if i != -1 {
// Add truncated 'folder' paths
keys = appendIfMissing(keys, string(key[:i+1]))
}
}

sort.Strings(keys)

return keys, nil
}
77 changes: 8 additions & 69 deletions physical/mysql_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package physical

import (
"database/sql"
"fmt"
"os"
"testing"

Expand All @@ -28,61 +26,6 @@ func TestMySQLBackend(t *testing.T) {
username := os.Getenv("MYSQL_USERNAME")
password := os.Getenv("MYSQL_PASSWORD")

// Create MySQL handle for the database.
db, err := sql.Open("mysql", username+":"+password+"@tcp("+address+")/"+database)

if err != nil {
t.Fatalf("Failed to open an handler with database: %v", err)
}
defer db.Close()

// Prepare statement for creating table.
create_stmt := "CREATE TABLE IF NOT EXISTS test.square (num int, sqr int, PRIMARY KEY (num))"
stmtCrt, err := db.Prepare(create_stmt)
if err != nil {
t.Fatalf("Failed to prepare statement: %v", err)
}
defer stmtCrt.Close()

// Create table
_, err = stmtCrt.Exec()
if err != nil {
t.Fatalf("Failed to create table: %v", err)
}

// Prepare statement for inserting data.
insert_stmt := "INSERT INTO test.square VALUES( ?, ? ) ON DUPLICATE KEY UPDATE sqr=VALUES(sqr)"
stmtIns, err := db.Prepare(insert_stmt)
if err != nil {
t.Fatalf("Failed to prepare statement: %v", err)
}
defer stmtIns.Close()

// Prepare statement for reading data.
select_stmt := "SELECT sqr FROM test.square WHERE num = ?"
stmtOut, err := db.Prepare(select_stmt)
if err != nil {
t.Fatalf("Failed to prepare statement: %v", err)
}
defer stmtOut.Close()

// Insert square numbers for 0-24 in the database
for i := 0; i < 25; i++ {
_, err = stmtIns.Exec(i, (i * i)) // Insert tuples (i, i^2)
if err != nil {
t.Fatalf("Failed to insert data: %v", err)
}
}

var square int

// Query the square-number of 13
err = stmtOut.QueryRow(13).Scan(&square)
if err != nil {
t.Fatalf("Failed to query data: %v", err)
}
fmt.Printf("The square number of 13 is: %d", square)

// Run vault tests
b, err := NewBackend("mysql", map[string]string{
"address": address,
Expand All @@ -96,19 +39,15 @@ func TestMySQLBackend(t *testing.T) {
t.Fatalf("Failed to create new backend: %v", err)
}

defer func() {
mysql := b.(*MySQLBackend)
_, err := mysql.client.Exec("DROP TABLE " + mysql.dbTable)
if err != nil {
t.Fatalf("Failed to drop table: %v", err)
}
}()

testBackend(t, b)
testBackend_ListPrefix(t, b)

// Drop table after running tests
drop_stmt := "DROP TABLE " + database + "." + table
stmt, err := db.Prepare(drop_stmt)
if err != nil {
t.Fatalf("Failed to prepare statement: %v", err)
}
defer stmt.Close()

_, err = stmt.Exec()
if err != nil {
t.Fatalf("Failed to drop table: %v", err)
}
}
17 changes: 17 additions & 0 deletions website/source/docs/config/index.html.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ durability, etc.
* `s3` - Store data within an S3 bucket [S3](http://aws.amazon.com/s3/).
This backend does not support HA.

* `mysql` - Store data within MySQL. This backend does not support HA.

* `inmem` - Store data in-memory. This is only really useful for
development and experimentation. Data is lost whenever Vault is
restarted.
Expand Down Expand Up @@ -143,6 +145,21 @@ For S3, the following options are supported:

* `region` (optional) - The AWS region. It can be sourced from the AWS_DEFAULT_REGION environment variable and will default to "us-east-1" if not specified.

#### Backend Reference: MySQL

The MySQL backend has the following options:

* `username` (required) - The MySQL username to connect with.

* `password` (required) - The MySQL password to connect with.

* `address` (optional) - The address of the MySQL host. Defaults to
"127.0.0.1:3306.

* `database` (optional) - The name of the database to use. Defaults to "vault".

* `table` (optional) - The name of the table to use. Defaults to "vault".

#### Backend Reference: Inmem

The in-memory backend has no configuration options.
Expand Down

0 comments on commit 46ba8d1

Please sign in to comment.