feat: child delete implementation

Dockerfile.build

@@ -1,4 +1,4 @@
-FROM golang:1.11.5
+FROM golang:1.12.5
 
 ADD . /go/src/github.com/thoas/picfit
 

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM golang:1.11.5
+FROM golang:1.12.5
 
 ADD . /go/src/github.com/thoas/picfit
 

README.rst

@@ -714,10 +714,55 @@ By default the format will be chosen in this order:
 Options
 =======
 
+Deletion
+--------
+
+Deletion is disabled by default for security reason, you can enable
+it in your config:
+
+``config.json``
+
+.. code-block:: json
+
+    {
+      "options": {
+        "enable_delete": true
+      }
+    }
+
+You will be able to delete root image and its children, for example if you upload an image with
+the file path `/foo/bar.png`, you can delete the main image on stockage by sending the following HTTP request:
+
+
+::
+
+   DELETE https://localhost:3001/foo/bar.png
+
+or delete a child:
+
+::
+
+   DELETE https://localhost:3001/display/thumbnail/100x100/foo/bar.png
+
+If you want to delete the main image and cascade its children, you can enable it in your config:
+
+``config.json``
+
+.. code-block:: json
+
+    {
+      "options": {
+        "enable_delete": true,
+        "enable_cascade_delete": true
+      }
+    }
+
+when a new image will be processed, it will be linked to the main image and stored in the kvstore.
+
 Upload
 ------
 
-The upload handler is disabled by default for security reason, you can enable
+Upload is disabled by default for security reason, you can enable
 it in your config:
 
 ``config.json``
@@ -733,9 +778,26 @@ it in your config:
 Stats
 -----
 
-The stats middleware is disabled by default, you can enable it your config.
+Stats are disabled by default, you can enable them in your config.
+
+``config.json``
+
+.. code-block:: json
+
+    {
+      "options": {
+        "enable_stats": true
+      }
+    }
+
+It will store various information about your web application (response time, status code count, etc.).
+
+To access these information, you can visit: http://localhost:3001/sys/stats
 
-It will store various information about your web application (response time, status code count, etc.)
+Health
+------
+
+Health is disabled by default, you can enable it in your config.
 
 ``config.json``
 
@@ -747,6 +809,49 @@ It will store various information about your web application (response time, sta
       }
     }
 
+It will show various internal information about the Go runtime (memory, allocations, etc.).
+
+To access these information, you can visit: http://localhost:3001/sys/health
+
+Profiler
+--------
+
+Profiler is disabled by default, you can enable it in your config.
+
+``config.json``
+
+.. code-block:: json
+
+    {
+      "options": {
+        "enable_pprof": true
+      }
+    }
+
+It will start pprof_ then use the pprof tool to look at the heap profile:
+
+::
+
+   go tool pprof http://localhost:3001/debug/pprof/heap
+
+Or to look at a 30-second CPU profile:
+
+::
+
+   go tool pprof http://localhost:3001/debug/pprof/profile
+
+Or to look at the goroutine blocking profile, after calling runtime.SetBlockProfileRate in your program:
+
+::
+
+   go tool pprof http://localhost:3001/debug/pprof/block
+
+Or to collect a 5-second execution trace:
+
+::
+
+   wget http://localhost:3001/debug/pprof/trace?seconds=5
+
 Logging
 -------
 
@@ -792,6 +897,24 @@ preserve aspect ratio.
       }
     }
 
+IP Address restriction
+----------------------
+
+You can restrict access to upload, stats, health, delete and pprof endpoints by enabling
+restriction in your config:
+
+``config.json``
+
+.. code-block:: json
+
+    {
+      "options": {
+        "allowed_ip_addresses": [
+          "127.0.0.1"
+        ]
+      }
+    }
+
 Deployment
 ==========
 
@@ -846,3 +969,4 @@ Thanks to these beautiful projects.
 .. _sentry: https://github.com/getsentry/sentry
 .. _raven: https://github.com/getsentry/raven-go
 .. _httpie: https://github.com/jakubroztocil/httpie
+.. _pprof: https://blog.golang.org/profiling-go-programs

application/application.go

@@ -8,6 +8,7 @@ import (
 
 	conv "github.com/cstockton/go-conv"
 	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
 
 	"github.com/thoas/picfit/config"
 	"github.com/thoas/picfit/engine"
@@ -55,30 +56,28 @@ func Load(cfg *config.Config) (context.Context, error) {
 
 // Store stores an image file with the defined filepath
 func Store(ctx context.Context, filepath string, i *image.ImageFile) error {
-	l := logger.FromContext(ctx)
-
-	cfg := config.FromContext(ctx)
-
-	k := kvstore.FromContext(ctx)
+	var (
+		l   = logger.FromContext(ctx)
+		cfg = config.FromContext(ctx)
+		k   = kvstore.FromContext(ctx)
+	)
 
 	err := i.Save()
-
 	if err != nil {
 		return err
 	}
 
 	l.Infof("Save thumbnail %s to storage", i.Filepath)
 
 	err = k.Set(i.Key, i.Filepath)
-
 	if err != nil {
 		return err
 	}
 
 	l.Infof("Save key %s => %s to kvstore", i.Key, i.Filepath)
 
 	// Write children info only when we actually want to be able to delete things.
-	if cfg.Options.EnableDelete {
+	if cfg.Options.EnableCascadeDelete {
 		parentKey := hash.Tokey(filepath)
 
 		parentKey = fmt.Sprintf("%s:children", parentKey)
@@ -94,13 +93,52 @@ func Store(ctx context.Context, filepath string, i *image.ImageFile) error {
 	return nil
 }
 
+// DeleteChild remove a child from kvstore and storage
+func DeleteChild(ctx context.Context, key string) error {
+	var (
+		k     = kvstore.FromContext(ctx)
+		store = storage.DestinationFromContext(ctx)
+		l     = logger.FromContext(ctx)
+	)
+
+	// Now, every child is a hash which points to a key/value pair in
+	// KVStore which in turn points to a file in dst storage.
+	dstfileRaw, err := k.Get(key)
+	if err != nil {
+		return errors.Wrapf(err, "unable to retrieve key %s", key)
+	}
+
+	if dstfileRaw != nil {
+		dstfile, err := conv.String(dstfileRaw)
+		if err != nil {
+			return errors.Wrapf(err, "unable to cast %v to string", dstfileRaw)
+		}
+
+		// And try to delete it all.
+		err = store.Delete(dstfile)
+		if err != nil {
+			return errors.Wrapf(err, "unable to delete %s on storage", dstfile)
+		}
+	}
+
+	err = k.Delete(key)
+	if err != nil {
+		return errors.Wrapf(err, "unable to delete key %s", key)
+	}
+
+	l.Infof("Deleting child %s", key)
+
+	return nil
+}
+
 // Delete removes a file from kvstore and storage
 func Delete(ctx context.Context, filepath string) error {
-	k := kvstore.FromContext(ctx)
-
-	l := logger.FromContext(ctx)
+	var (
+		k = kvstore.FromContext(ctx)
+		l = logger.FromContext(ctx)
+	)
 
-	l.Infof("Deleting source storage file: %s", filepath)
+	l.Infof("Deleting %s on source storage", filepath)
 
 	sourceStorage := storage.SourceFromContext(ctx)
 
@@ -111,9 +149,8 @@ func Delete(ctx context.Context, filepath string) error {
 	}
 
 	err := sourceStorage.Delete(filepath)
-
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "unable to delete %s on source storage", filepath)
 	}
 
 	parentKey := hash.Tokey(filepath)
@@ -122,7 +159,7 @@ func Delete(ctx context.Context, filepath string) error {
 
 	exists, err := k.Exists(childrenKey)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "unable to verify if %s exists", childrenKey)
 	}
 
 	if !exists {
@@ -134,7 +171,7 @@ func Delete(ctx context.Context, filepath string) error {
 	// Get the list of items to cleanup.
 	children, err := k.GetSlice(childrenKey)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "unable to retrieve children set %s", childrenKey)
 	}
 
 	if children == nil {
@@ -143,49 +180,26 @@ func Delete(ctx context.Context, filepath string) error {
 		return nil
 	}
 
-	store := storage.DestinationFromContext(ctx)
+	l.Infof("Children %v from parent %s will be deleted", children, parentKey)
 
 	for _, s := range children {
 		key, err := conv.String(s)
 		if err != nil {
 			return err
 		}
 
-		// Now, every child is a hash which points to a key/value pair in
-		// KVStore which in turn points to a file in dst storage.
-		dstfileRaw, err := k.Get(key)
-		if err != nil {
-			return err
-		}
-
-		dstfile, err := conv.String(dstfileRaw)
-		if err != nil {
-			return err
-		}
-
-		// And try to delete it all.
-		err = store.Delete(dstfile)
-
-		if err != nil {
-			return err
-		}
-
-		err = k.Delete(key)
-
+		err = DeleteChild(ctx, key)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "unable to delete child %s", key)
 		}
-
-		l.Infof("Deleting child %s and its entry %s", dstfile, key)
 	}
 
 	// Delete them right away, we don't care about them anymore.
 	l.Infof("Deleting children set %s", childrenKey)
 
 	err = k.Delete(childrenKey)
-
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "unable to delete key %s", childrenKey)
 	}
 
 	return nil
@@ -243,21 +257,22 @@ func fileFromStorage(c *gin.Context, l logger.Logger, storeKey string, imageKey
 }
 
 func processImage(c *gin.Context, l logger.Logger, storeKey string, async bool) (*image.ImageFile, error) {
-	cfg := config.FromContext(c)
-	destStorage := storage.DestinationFromContext(c)
+	var (
+		cfg         = config.FromContext(c)
+		destStorage = storage.DestinationFromContext(c)
+		s           = storage.SourceFromContext(c)
+		filepath    string
+		err         error
+	)
 
 	file := &image.ImageFile{
 		Key:     storeKey,
 		Storage: destStorage,
 		Headers: map[string]string{},
 	}
 
-	var filepath string
 	qs := c.MustGet("parameters").(map[string]interface{})
 
-	var err error
-	s := storage.SourceFromContext(c)
-
 	u, exists := c.Get("url")
 	if exists {
 		file, err = image.FromURL(u.(*url.URL), cfg.Options.DefaultUserAgent)