Skip to content

Commit

Permalink
Add PostgreSQL support (#24)
Browse files Browse the repository at this point in the history
* add postgresql support

* go mod tidy and remove any sqlite3 references

* fix build and push dev images to github package registry

* fix github workflow file

* only push if tests pass

* push latest and build timestamp

* use correct paths

* fix bundle name

* Update README
  • Loading branch information
uhthomas committed May 27, 2020
1 parent e8189be commit 51d0662
Show file tree
Hide file tree
Showing 13 changed files with 142 additions and 52 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/test.yml
Expand Up @@ -13,4 +13,10 @@ jobs:
mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel"
chmod +x "${GITHUB_WORKSPACE}/bin/bazel"
- run: bazel build //...
- run: bazel test //...
- run: bazel test //...
- uses: azure/docker-login@v1
with:
login-server: docker.pkg.github.com
username: $GITHUB_ACTOR
password: ${{ secrets.GITHUB_TOKEN }}
- run: bazel run //cmd/kipp:push_github
23 changes: 18 additions & 5 deletions README.md
Expand Up @@ -5,19 +5,32 @@
## Getting started
The easiest way to get started with kipp, is by using the image published to
[Docker Hub](https://hub.docker.com/repository/docker/uhthomas/kipp). The
service is then available simply by running
service is then available simply by running:
```
docker pull uhthomas/kipp
docker run uhthomas/kipp
```

## Databases
* [Badger](https://github.com/dgraph-io/badger)
Databases can be configured using the `--database` flag. The flag requires
the input be parsable as a URL. See the [url.Parse](https://golang.org/pkg/net/url/#Parse)
docs for more info.

### [Badger](https://github.com/dgraph-io/badger)
Badger is a fast, embedded database which is great for single instances.

### SQL
Kipp uses a generic SQL driver, but currently only loads:
* [PostgreSQL](https://www.postgresql.org/)

As long as a database supports Go's [sql](https://golang.org/pkg/database/sql/)
package, it can be used. Please file an issue for requests.

## File systems
File systems can be configured using the `--filesystem` flag. The flag requires
the input be parsable as a URL. See the [url.Parse](https://golang.org/pkg/net/url/#Parse)
docs for more info.

### Local (your local file system)
The local filesystem does not require any special formatting, and can be used
like a regular path such
Expand All @@ -43,6 +56,8 @@ The `endpoint` is optional, and will use the default AWS endpoint if not present
This is useful for using S3-compatible services such as:
* [Google Cloud Storage](https://cloud.google.com/storage) - storage.googleapis.com
* [Linode Object Storage](https://www.linode.com/products/object-storage/) - linodeobjects.com
* [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) - backblazeb2.com
* [DigitalOcean Spaces](https://www.digitalocean.com/products/spaces/) - digitaloceanspaces.com
* ... etc

#### Policy
Expand All @@ -53,10 +68,8 @@ Required actions:

This is subject to change in future as more features are added.

### [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) (experimental)

## Building from source
Kipp is built, tested and compiled using [Bazel](https://bazel.build). To run
Kipp builds, tests and compiles using [Bazel](https://bazel.build). To run/build
locally with bazel:
```
git clone git@github.com:uhthomas/kipp
Expand Down
7 changes: 7 additions & 0 deletions WORKSPACE
Expand Up @@ -450,3 +450,10 @@ go_repository(
sum = "h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=",
version = "v0.5.3",
)

go_repository(
name = "com_github_lib_pq",
importpath = "github.com/lib/pq",
sum = "h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw=",
version = "v1.5.2",
)
37 changes: 29 additions & 8 deletions cmd/kipp/BUILD.bazel
Expand Up @@ -12,9 +12,10 @@ go_library(
visibility = ["//visibility:private"],
deps = [
"//:go_default_library",
"//database/badger:go_default_library",
"//internal/databaseutil:go_default_library",
"//internal/filesystemutil:go_default_library",
"@com_github_alecthomas_units//:go_default_library",
"@com_github_lib_pq//:go_default_library",
],
)

Expand All @@ -32,22 +33,42 @@ go_image(
visibility = ["//visibility:private"],
deps = [
"//:go_default_library",
"//database/badger:go_default_library",
"//internal/databaseutil:go_default_library",
"//internal/filesystemutil:go_default_library",
"@com_github_alecthomas_units//:go_default_library",
"@com_github_lib_pq//:go_default_library",
],
goarch = "amd64",
goos = "linux",
data = ["//:web"],
)

load("@io_bazel_rules_docker//docker:docker.bzl", "docker_push")
load("@io_bazel_rules_docker//docker:docker.bzl", "docker_bundle")

docker_bundle(
name = "kipp_bundle",
images = {
"index.docker.io/uhthomas/kipp:latest": ":kipp",
"index.docker.io/uhthomas/kipp:{BUILD_TIMESTAMP}": ":kipp",
},
)

docker_bundle(
name = "kipp_bundle_github",
images = {
"docker.pkg.github.com/uhthomas/kipp/kipp:latest": ":kipp",
"docker.pkg.github.com/uhthomas/kipp/kipp:{BUILD_TIMESTAMP}": ":kipp",
},
)

load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")

docker_push(
name = "push",
image = ":kipp",
registry = "index.docker.io",
repository = "uhthomas/kipp",
tag = "latest",
tags = ["{BUILD_TIMESTAMP}"],
bundle = "kipp_bundle",
)

docker_push(
name = "push_github",
bundle = "kipp_bundle_github",
)
11 changes: 6 additions & 5 deletions cmd/kipp/serve.go
Expand Up @@ -10,14 +10,15 @@ import (
"net/http"
"time"

_ "github.com/lib/pq"
"github.com/uhthomas/kipp"
"github.com/uhthomas/kipp/database/badger"
"github.com/uhthomas/kipp/internal/filesystemutil"
filesystemutil "github.com/uhthomas/kipp/internal/databaseutil"
databaseutil "github.com/uhthomas/kipp/internal/filesystemutil"
)

func serve(ctx context.Context) error {
addr := flag.String("addr", ":80", "listen addr")
dsn := flag.String("dsn", "badger", "data source name")
dbf := flag.String("database", "badger", "database - see docs for more information")
fsf := flag.String("filesystem", "files", "filesystem - see docs for more information")
web := flag.String("web", "web", "web directory")
limit := flagBytesValue("limit", 150<<20, "upload limit")
Expand All @@ -37,9 +38,9 @@ func serve(ctx context.Context) error {
return fmt.Errorf("parse filesystem: %w", err)
}

db, err := badger.Open(*dsn)
db, err := databaseutil.Parse(ctx, *dbf)
if err != nil {
return fmt.Errorf("open badger: %w", err)
return fmt.Errorf("parse database: %w", err)
}
defer db.Close(ctx)

Expand Down
4 changes: 2 additions & 2 deletions database/sqlite3/BUILD.bazel → database/sql/BUILD.bazel
Expand Up @@ -2,8 +2,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["sqlite3.go"],
importpath = "github.com/uhthomas/kipp/database/sqlite3",
srcs = ["sql.go"],
importpath = "github.com/uhthomas/kipp/database/sql",
visibility = ["//visibility:public"],
deps = ["//database:go_default_library"],
)
10 changes: 5 additions & 5 deletions database/sqlite3/sqlite3.go → database/sql/sql.go
@@ -1,4 +1,4 @@
package sqlite3
package sql

import (
"context"
Expand All @@ -9,7 +9,7 @@ import (
"github.com/uhthomas/kipp/database"
)

// A Database is a wrapper around a sqlite3 db which provides high level
// A Database is a wrapper around a sql db which provides high level
// functions defined in database.Database.
type Database struct {
db *sql.DB
Expand All @@ -30,9 +30,9 @@ const initQuery = `CREATE TABLE IF NOT EXISTS entries (
CREATE UNIQUE INDEX IF NOT EXISTS idx_slug ON entries (slug)`

// Open opens a new sqlite3 database and prepares relevant statements.
func Open(ctx context.Context, name string) (*Database, error) {
db, err := sql.Open("sqlite3", name)
// Open opens a new sql database and prepares relevant statements.
func Open(ctx context.Context, driver, name string) (*Database, error) {
db, err := sql.Open(driver, name)
if err != nil {
return nil, fmt.Errorf("sql open: %w", err)
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Expand Up @@ -8,5 +8,6 @@ require (
github.com/dgraph-io/badger/v2 v2.0.3
github.com/dgraph-io/ristretto v0.0.2 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/lib/pq v1.5.2
github.com/zeebo/blake3 v0.0.1
)
2 changes: 2 additions & 0 deletions go.sum
Expand Up @@ -42,6 +42,8 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw=
github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
Expand Down
15 changes: 15 additions & 0 deletions internal/databaseutil/BUILD.bazel
@@ -0,0 +1,15 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["parse.go"],
importpath = "github.com/uhthomas/kipp/internal/databaseutil",
visibility = ["//:__subpackages__"],
deps = [
"//filesystem:go_default_library",
"//filesystem/local:go_default_library",
"//filesystem/s3:go_default_library",
"@com_github_aws_aws_sdk_go//aws:go_default_library",
"@com_github_aws_aws_sdk_go//aws/credentials:go_default_library",
],
)
37 changes: 37 additions & 0 deletions internal/databaseutil/parse.go
@@ -0,0 +1,37 @@
package filesystemutil

import (
"fmt"
"net/url"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/uhthomas/kipp/filesystem"
"github.com/uhthomas/kipp/filesystem/local"
"github.com/uhthomas/kipp/filesystem/s3"
)

// Parse parses s, and will create the appropriate filesystem for the scheme.
func Parse(s string) (filesystem.FileSystem, error) {
u, err := url.Parse(s)
if err != nil {
return nil, err
}
switch u.Scheme {
case "":
return local.New(u.Path)
// s3 follows the form:
// s3://key:token@region/bucket?endpoint=v
case "s3":
c := &aws.Config{Region: &u.Host}
if u.User != nil {
p, _ := u.User.Password()
c.Credentials = credentials.NewStaticCredentials(u.User.Username(), p, "")
}
if e := u.Query().Get("endpoint"); e != "" {
c.Endpoint = &e
}
return s3.New(u.Path, c)
}
return nil, fmt.Errorf("invalid scheme: %s", u.Scheme)
}
8 changes: 3 additions & 5 deletions internal/filesystemutil/BUILD.bazel
Expand Up @@ -6,10 +6,8 @@ go_library(
importpath = "github.com/uhthomas/kipp/internal/filesystemutil",
visibility = ["//:__subpackages__"],
deps = [
"//filesystem:go_default_library",
"//filesystem/local:go_default_library",
"//filesystem/s3:go_default_library",
"@com_github_aws_aws_sdk_go//aws:go_default_library",
"@com_github_aws_aws_sdk_go//aws/credentials:go_default_library",
"//database:go_default_library",
"//database/badger:go_default_library",
"//database/sql:go_default_library",
],
)
31 changes: 10 additions & 21 deletions internal/filesystemutil/parse.go
@@ -1,37 +1,26 @@
package filesystemutil
package databaseutil

import (
"context"
"fmt"
"net/url"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/uhthomas/kipp/filesystem"
"github.com/uhthomas/kipp/filesystem/local"
"github.com/uhthomas/kipp/filesystem/s3"
"github.com/uhthomas/kipp/database"
"github.com/uhthomas/kipp/database/badger"
"github.com/uhthomas/kipp/database/sql"
)

// Parse parses s, and will create the appropriate filesystem for the scheme.
func Parse(s string) (filesystem.FileSystem, error) {
// Parse parses s, and will create the appropriate database for the scheme.
func Parse(ctx context.Context, s string) (database.Database, error) {
u, err := url.Parse(s)
if err != nil {
return nil, err
}
switch u.Scheme {
case "":
return local.New(u.Path)
// s3 follows the form:
// s3://key:token@region/bucket?endpoint=v
case "s3":
c := &aws.Config{Region: &u.Host}
if u.User != nil {
p, _ := u.User.Password()
c.Credentials = credentials.NewStaticCredentials(u.User.Username(), p, "")
}
if e := u.Query().Get("endpoint"); e != "" {
c.Endpoint = &e
}
return s3.New(u.Path, c)
return badger.Open(u.Path)
case "postgresql":
return sql.Open(ctx, "postgresql", u.String())
}
return nil, fmt.Errorf("invalid scheme: %s", u.Scheme)
}

0 comments on commit 51d0662

Please sign in to comment.