Skip to content

Commit

Permalink
Merge pull request #1 from Shivam010/pfs
Browse files Browse the repository at this point in the history
Add proxied file system blob implementation with its route handler
  • Loading branch information
Shivam010 committed Jan 27, 2022
2 parents 403f04b + 0103596 commit d946b12
Show file tree
Hide file tree
Showing 15 changed files with 383 additions and 16 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,6 @@ jobs:
S3_BASE_URL: ${{ secrets.S3_BASE_URL }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}


- name: Test pfsblob
run: go run github.com/Shivam010/upload/pfsblob/tests
38 changes: 34 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,44 @@
# upload
# upload

[![Build](https://github.com/Shivam010/upload/actions/workflows/build.yml/badge.svg)](https://github.com/Shivam010/upload/actions/workflows/build.yml)
[![Go Report Card](https://goreportcard.com/badge/github.com/Shivam010/upload?dropcache)](https://goreportcard.com/report/github.com/Shivam010/upload)
[![Go Reference](https://pkg.go.dev/badge/github.com/Shivam010/upload)](https://pkg.go.dev/github.com/Shivam010/upload)
[![License](https://img.shields.io/badge/license-MIT-mildgreen.svg)](https://github.com/Shivam010/upload/blob/master/LICENSE)
[![GitHub release](https://img.shields.io/github/release/Shivam010/upload.svg)](https://github.com/Shivam010/upload/releases)

Package upload provides an easy and portable way to interact with CDN, buckets or blobs within any cloud/local storage location. And provides methods to read or write or upload or delete files to blob storage on GCP, AWS, Azure, in-memory, local and more.
Package upload provides an easy and portable way to interact with CDN, buckets or blobs within any cloud/local storage
location. And provides methods to read or write or upload or delete files to blob storage on GCP, AWS, Azure, in-memory,
local and more.

It wraps `gocloud.dev/blob` (https://github.com/google/go-cloud/tree/master/blob) for further simplicity.

### Example
## Proxied File system with HTTP Router

You can store the files in using file system blob and can serve them using http file server. Upload library provides
everything you need to do that in-built. Just use `pfsblob` as your file system handler. See the example
in [pfsblob/tests](./pfsblob/tests).

### URL format for pfsblob

1. URL Scheme: `pfs`
2. URL Host: Your domain name (without protocol) e.g. `localhost:8080` or `example.com`
3. URL Path: Your storage directory in which you want to manage your files e.g. `/tmp/files`
4. Domain Protocol: If your domain uses secure `https` protocol, set a query parameter: `?secure=true`
5. Route: By default, files will be served at the root of the domain, if you want to change it, use query
parameter: `?route=static`

Complete Query will look something like these:

1. **`pfs://localhost:8080/tmp/files?secure=false`** <br/>
This will serve files at `http://localhost:8080/...` <br/>
e.g. the link for the file `/tmp/files/image.png` will be `http://localhost:8080/image.png`

2. **`pfs://example.com/tmp/files?secure=true&route=public`** <br/>
This will serve files at `https://example.com/public/...` <br/>
e.g. the link for the file `/tmp/files/image.png` will be `https://example.com/public/image.png`

## Example

```go
package main

Expand Down Expand Up @@ -62,5 +91,6 @@ func main() {
}
```

### License
## License

This project is licensed under the [MIT License](./LICENSE)
51 changes: 46 additions & 5 deletions bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import (
"fmt"
"io"
"net/url"
"strings"

pfs "github.com/Shivam010/upload/pfsblob"
"gocloud.dev/blob"
file "gocloud.dev/blob/fileblob"
gcs "gocloud.dev/blob/gcsblob"
Expand All @@ -22,13 +24,15 @@ const (
FileSystem
GoogleCloud
AmazonWebServices
ProxiedFileSystem
)

var ProviderName = map[Provider]string{
InMemory: "In-Memory",
FileSystem: "Local File System",
GoogleCloud: "Google Cloud Console",
AmazonWebServices: "Amazon Web Services",
ProxiedFileSystem: "Proxied File System",
}

func (p Provider) String() string {
Expand All @@ -38,14 +42,14 @@ func (p Provider) String() string {
type Bucket struct {
url string
name string
region string
provider Provider
bucket *blob.Bucket
metadata map[string]string
}

// NewBucket will return the blob bucket using the provided bucket url
func NewBucket(bucket string) *Bucket {
b := &Bucket{url: bucket}
b := &Bucket{url: bucket, metadata: map[string]string{}}
b.parse()
return b
}
Expand Down Expand Up @@ -85,7 +89,25 @@ func (b *Bucket) parse() {
case s3.Scheme:
b.provider = AmazonWebServices
b.name = u.Host
b.region = u.Query().Get("region")
b.metadata["region"] = u.Query().Get("region")
case pfs.Scheme:
b.provider = ProxiedFileSystem
// storage directory of the file system
b.metadata["storage"] = u.Path
// listening route region
route := strings.Trim(u.Query().Get("route"), "/")
b.metadata["route"] = route

// name of the bucket (for the link purposes)
b.name = "http://"
isSecure := u.Query().Get("secure")
if isSecure == "true" {
b.name = "https://"
}
b.name += u.Host
if route != "" {
b.name += "/" + route
}
default:
b.name = u.Host
b.url = mem.Scheme + "://" + b.name
Expand All @@ -98,11 +120,26 @@ func (b *Bucket) Provider() Provider {
return b.provider
}

// String implements stringer on Bucket
func (b *Bucket) String() string {
return b.Provider().String() + " Bucket"
}

// Name returns name of the bucket
func (b *Bucket) Name() string {
return b.name
}

// URL returns the url of the bucket
func (b *Bucket) URL() string {
return b.url
}

// GetMetadata returns the value stored inside the key of the metadata of bucket
func (b *Bucket) GetMetadata(key string) string {
return b.metadata[key]
}

// OpenContext opens a new bucket connection
func (b *Bucket) OpenContext(ctx context.Context) (err error) {
b.parse()
Expand Down Expand Up @@ -156,7 +193,9 @@ func (b *Bucket) GetUrl(name string) string {
case GoogleCloud:
return fmt.Sprintf("https://storage.googleapis.com/%v/%v", b.name, name)
case AmazonWebServices:
return fmt.Sprintf("https://%v.s3.%v.amazonaws.com/%v", b.name, b.region, name)
return fmt.Sprintf("https://%v.s3.%v.amazonaws.com/%v", b.name, b.GetMetadata("region"), name)
case ProxiedFileSystem:
return b.name + "/" + name
}
return b.name + "/" + name
}
Expand All @@ -172,7 +211,9 @@ func (b *Bucket) GetName(link string) string {
case GoogleCloud:
prefix = fmt.Sprintf("https://storage.googleapis.com/%v/", b.name)
case AmazonWebServices:
prefix = fmt.Sprintf("https://%v.s3.%v.amazonaws.com/", b.name, b.region)
prefix = fmt.Sprintf("https://%v.s3.%v.amazonaws.com/", b.name, b.GetMetadata("region"))
case ProxiedFileSystem:
prefix = b.name + "/"
}
if len(link) > len(prefix) {
return link[len(prefix):]
Expand Down
61 changes: 56 additions & 5 deletions bucket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@ func GetTestData(t *testing.T) []unitData {
ioList: []struct{ input, output string }{
{
input: "number/one.in",
output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/")+"number/one.in",
output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/") + "number/one.in",
},
{
input: "two",
output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/")+"two",
output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/") + "two",
},
},
},
Expand All @@ -61,11 +61,53 @@ func GetTestData(t *testing.T) []unitData {
ioList: []struct{ input, output string }{
{
input: "number/one.in",
output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/")+"number/one.in",
output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/") + "number/one.in",
},
{
input: "two",
output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/")+"two",
output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/") + "two",
},
},
},
{
name: ProxiedFileSystem.String(),
bucket: "pfs://localhost:8080" + pwd() + "/bin?route=web/file/srv",
ioList: []struct{ input, output string }{
{
input: "number/three.in",
output: "http://localhost:8080/web/file/srv/number/three.in",
},
{
input: "four",
output: "http://localhost:8080/web/file/srv/four",
},
},
},
{
name: ProxiedFileSystem.String() + " (Secure)",
bucket: "pfs://example.com" + pwd() + "/bin/?route=web/file/srv&secure=true",
ioList: []struct{ input, output string }{
{
input: "number/three.in",
output: "https://example.com/web/file/srv/number/three.in",
},
{
input: "four",
output: "https://example.com/web/file/srv/four",
},
},
},
{
name: ProxiedFileSystem.String() + " (without route)",
bucket: "pfs://example.com" + pwd() + "/bin",
ioList: []struct{ input, output string }{
{
input: "number/three.in",
output: "http://example.com/number/three.in",
},
{
input: "four",
output: "http://example.com/four",
},
},
},
Expand All @@ -85,10 +127,14 @@ func TestUpload(t *testing.T) {
continue
}
t.Log(bucket.Provider())
if bucket.Provider() == AmazonWebServices {
t.Skip("Credentials are removed")
}
// upload content
got, err := bucket.WriteAll(ctx, io.input, []byte(io.input))
if err != nil {
t.Errorf("WriteAll(%v), got: %v \n", io.input, err)
continue
}
if got != io.output {
t.Errorf("GetUrl(%v), got: %v want: %v \n", io.input, got, io.output)
Expand All @@ -97,6 +143,7 @@ func TestUpload(t *testing.T) {
con, err := bucket.ReadAll(ctx, bucket.GetName(io.output))
if err != nil {
t.Errorf("ReadAll(%v), got: %v \n", io.output, err)
continue
}
if string(con) != io.input {
t.Errorf("Content(%v), got: %s want: %v \n", io.output, con, io.input)
Expand All @@ -115,10 +162,14 @@ func TestUpload(t *testing.T) {
for _, io := range tt.ioList {
bucket := NewBucket(tt.bucket)
t.Log(bucket.Provider())
if bucket.Provider() == AmazonWebServices {
t.Skip("Credentials are removed")
}
// upload content
got, err := bucket.WriteAll(ctx, io.input, []byte(io.input))
if err != nil {
t.Errorf("WriteAll(%v), got: %v \n", io.input, err)
continue
}
if got != io.output {
t.Errorf("GetUrl(%v), got: %v want: %v \n", io.input, got, io.output)
Expand All @@ -127,14 +178,14 @@ func TestUpload(t *testing.T) {
r, err := bucket.Reader(ctx, bucket.GetName(io.output))
if err != nil {
t.Errorf("Reader(%v), got: %v \n", io.output, err)
continue
}
con, err := ioutil.ReadAll(r)
if err != nil {
t.Errorf("ReadAll(%v), got: %v \n", io.output, err)
}
if err := r.Close(); err != nil {
t.Errorf("Close(%v), got: %v \n", io.output, err)
return
}
if string(con) != io.input {
t.Errorf("Content(%v), got: %s want: %v \n", io.output, con, io.input)
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,7 @@ module github.com/Shivam010/upload

go 1.16

require gocloud.dev v0.23.0
require (
gocloud.dev v0.23.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
)
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
53 changes: 53 additions & 0 deletions pfsblob/handler/route.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package handler

import (
"errors"
"github.com/Shivam010/upload"
"net/http"
"os"
)

func BucketRouteAndHandler(buck *upload.Bucket) (route string, handler func(http.ResponseWriter, *http.Request), err error) {
if buck == nil || buck.Provider() != upload.ProxiedFileSystem {
return "", nil, errors.New("handler: can only work on Proxied File System Bucket provider")
}
// Bucket region is alias for the route
// Bucket account is alias for the storage directory
route = "/" + buck.GetMetadata("route")
handlerFunc := http.StripPrefix(
route,
http.FileServer(wrappedFileSystem{
fs: http.Dir(buck.GetMetadata("storage")),
}),
).ServeHTTP

return route + "/",
func(w http.ResponseWriter, r *http.Request) {
// security and caching headers
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Referrer-Policy", "strict-origin")
w.Header().Set("X-XSS-Protection", "1; mode=block")
w.Header().Set("X-Frame-Options", "SAMEORIGIN")
w.Header().Set("Strict-Transport-Security", "max-age=31536000")
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
handlerFunc(w, r)
},
nil
}

type wrappedFileSystem struct {
fs http.FileSystem
}

func (w wrappedFileSystem) Open(path string) (http.File, error) {
file, err := w.fs.Open(path)
if err != nil {
return nil, err
}

st, err := file.Stat()
if st.IsDir() {
return nil, os.ErrNotExist
}
return file, nil
}

0 comments on commit d946b12

Please sign in to comment.