Skip to content
Permalink
Browse files

Add S3 backend (#156)

  • Loading branch information
mutantmonkey authored and andreimarcu committed Jan 25, 2019
1 parent 0fb5fa1 commit 5d9a93b1e285fe36b827ec74a2186f8623408b25
Showing with 737 additions and 440 deletions.
  1. +2 −0 Dockerfile
  2. +116 −27 backends/localfs/localfs.go
  3. +0 −5 backends/meta.go
  4. +0 −70 backends/metajson/metajson.go
  5. +192 −0 backends/s3/s3.go
  6. +8 −12 backends/storage.go
  7. +7 −12 delete.go
  8. +30 −16 display.go
  9. +1 −1 expiry.go
  10. +21 −16 fileserve.go
  11. +70 −0 helpers/archive.go
  12. +67 −0 helpers/helpers.go
  13. +3 −7 linx-cleanup/cleanup.go
  14. +0 −165 meta.go
  15. +27 −8 pages.go
  16. +16 −6 server.go
  17. +70 −10 server_test.go
  18. +23 −41 torrent.go
  19. +28 −0 torrent/torrent.go
  20. +3 −2 torrent_test.go
  21. +53 −42 upload.go
@@ -13,6 +13,8 @@ FROM alpine:3.8
COPY --from=build /go/bin/linx-server /usr/local/bin/linx-server

ENV GOPATH /go
ENV SSL_CERT_FILE /etc/ssl/cert.pem

COPY static /go/src/github.com/andreimarcu/linx-server/static/
COPY templates /go/src/github.com/andreimarcu/linx-server/templates/

@@ -1,63 +1,149 @@
package localfs

import (
"errors"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"time"

"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/helpers"
)

type LocalfsBackend struct {
basePath string
metaPath string
filesPath string
}

func (b LocalfsBackend) Delete(key string) error {
return os.Remove(path.Join(b.basePath, key))
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
}

func (b LocalfsBackend) Delete(key string) (err error) {
err = os.Remove(path.Join(b.filesPath, key))
if err != nil {
return
}
err = os.Remove(path.Join(b.metaPath, key))
return
}

func (b LocalfsBackend) Exists(key string) (bool, error) {
_, err := os.Stat(path.Join(b.basePath, key))
_, err := os.Stat(path.Join(b.filesPath, key))
return err == nil, err
}

func (b LocalfsBackend) Get(key string) ([]byte, error) {
return ioutil.ReadFile(path.Join(b.basePath, key))
func (b LocalfsBackend) Head(key string) (metadata backends.Metadata, err error) {
f, err := os.Open(path.Join(b.metaPath, key))
if os.IsNotExist(err) {
return metadata, backends.NotFoundErr
} else if err != nil {
return metadata, backends.BadMetadata
}
defer f.Close()

decoder := json.NewDecoder(f)

mjson := MetadataJSON{}
if err := decoder.Decode(&mjson); err != nil {
return metadata, backends.BadMetadata
}

metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size

return
}

func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
dst, err := os.Create(path.Join(b.basePath, key))
func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCloser, err error) {
metadata, err = b.Head(key)
if err != nil {
return 0, err
return
}

f, err = os.Open(path.Join(b.filesPath, key))
if err != nil {
return
}

return
}

func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error {
metaPath := path.Join(b.metaPath, key)

mjson := MetadataJSON{
DeleteKey: metadata.DeleteKey,
Mimetype: metadata.Mimetype,
ArchiveFiles: metadata.ArchiveFiles,
Sha256sum: metadata.Sha256sum,
Expiry: metadata.Expiry.Unix(),
Size: metadata.Size,
}

dst, err := os.Create(metaPath)
if err != nil {
return err
}
defer dst.Close()

encoder := json.NewEncoder(dst)
err = encoder.Encode(mjson)
if err != nil {
os.Remove(metaPath)
return err
}

return nil
}

func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) {
filePath := path.Join(b.filesPath, key)

dst, err := os.Create(filePath)
if err != nil {
return
}
defer dst.Close()

bytes, err := io.Copy(dst, r)
if bytes == 0 {
b.Delete(key)
return bytes, errors.New("Empty file")
os.Remove(filePath)
return m, backends.FileEmptyError
} else if err != nil {
b.Delete(key)
return bytes, err
os.Remove(filePath)
return m, err
}

return bytes, err
}
m.Expiry = expiry
m.DeleteKey = deleteKey
m.Size = bytes
m.Mimetype, _ = helpers.DetectMime(dst)
m.Sha256sum, _ = helpers.Sha256sum(dst)
m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst)

func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
return os.Open(path.Join(b.basePath, key))
}
err = b.writeMetadata(key, m)
if err != nil {
os.Remove(filePath)
return
}

func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
filePath := path.Join(b.basePath, key)
http.ServeFile(w, r, filePath)
return
}

func (b LocalfsBackend) Size(key string) (int64, error) {
fileInfo, err := os.Stat(path.Join(b.basePath, key))
fileInfo, err := os.Stat(path.Join(b.filesPath, key))
if err != nil {
return 0, err
}
@@ -68,7 +154,7 @@ func (b LocalfsBackend) Size(key string) (int64, error) {
func (b LocalfsBackend) List() ([]string, error) {
var output []string

files, err := ioutil.ReadDir(b.basePath)
files, err := ioutil.ReadDir(b.filesPath)
if err != nil {
return nil, err
}
@@ -80,6 +166,9 @@ func (b LocalfsBackend) List() ([]string, error) {
return output, nil
}

func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend {
return LocalfsBackend{
metaPath: metaPath,
filesPath: filesPath,
}
}
@@ -5,11 +5,6 @@ import (
"time"
)

type MetaBackend interface {
Get(key string) (Metadata, error)
Put(key string, metadata *Metadata) error
}

type Metadata struct {
DeleteKey string
Sha256sum string

This file was deleted.

0 comments on commit 5d9a93b

Please sign in to comment.
You can’t perform that action at this time.