Cleanup: Integrate expired files periodic cleanup

.gitignore

@@ -29,8 +29,8 @@ _testmain.go
 *.prof
 
 linx-server
-linx-cleanup
-linx-genkey
+linx-cleanup/linx-cleanup
+linx-genkey/linx-genkey
 files/
 meta/
 binaries/

README.md

@@ -92,6 +92,17 @@ allowhotlink = true
 | ```-force-random-filename``` | (optionally) force the use of random filenames
 | ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation 
 
+
+#### Cleaning up expired files
+When files expire, access is disabled immediately, but the files and metadata
+will persist on disk until someone attempts to access them. You can set the following option to run cleanup every few minutes. This can also be done using a separate utility found the linx-cleanup directory.
+
+
+|Option|Description
+|------|-----------
+| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)
+
+
 #### Require API Keys for uploads
 
 |Option|Description
@@ -127,26 +138,6 @@ The following storage backends are available:
 |------|-----------
 | ```-fastcgi``` | serve through fastcgi 
 
-
-Cleaning up expired files
--------------------------
-When files expire, access is disabled immediately, but the files and metadata
-will persist on disk until someone attempts to access them. If you'd like to
-automatically clean up files that have expired, you can use the included
-`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
-of scheduled task.
-
-You should be careful to ensure that only one instance of `linx-cleanup` runs at
-a time to avoid unexpected behavior. It does not implement any type of locking.
-
-
-|Option|Description
-|------|-----------
-| ```-filespath files/``` | Path to stored uploads (default is files/)
-| ```-nologs``` | (optionally) disable deletion logs in stdout
-| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)
-
-
 Deployment
 ----------
 Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).
@@ -207,4 +198,4 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 Author
 -------
-Andrei Marcu, http://andreim.net/
+Andrei Marcu, https://andreim.net/

linx-cleanup/cleanup.go → cleanup/cleanup.go

@@ -1,26 +1,14 @@
-package main
+package cleanup
 
 import (
-	"flag"
 	"log"
+	"time"
 
 	"github.com/andreimarcu/linx-server/backends/localfs"
 	"github.com/andreimarcu/linx-server/expiry"
 )
 
-func main() {
-	var filesDir string
-	var metaDir string
-	var noLogs bool
-
-	flag.StringVar(&filesDir, "filespath", "files/",
-		"path to files directory")
-	flag.StringVar(&metaDir, "metapath", "meta/",
-		"path to metadata directory")
-	flag.BoolVar(&noLogs, "nologs", false,
-		"don't log deleted files")
-	flag.Parse()
-
+func Cleanup(filesDir string, metaDir string, noLogs bool) {
 	fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir)
 
 	files, err := fileBackend.List()
@@ -44,3 +32,11 @@ func main() {
 		}
 	}
 }
+
+func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) {
+	c := time.Tick(minutes)
+	for range c {
+		Cleanup(filesDir, metaDir, noLogs)
+	}
+
+}

linx-cleanup/README.md

@@ -0,0 +1,19 @@
+
+linx-cleanup
+-------------------------
+When files expire, access is disabled immediately, but the files and metadata
+will persist on disk until someone attempts to access them. 
+
+If you'd like to automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type
+of scheduled task.
+
+You should be careful to ensure that only one instance of `linx-cleanup` runs at
+a time to avoid unexpected behavior. It does not implement any type of locking.
+
+
+|Option|Description
+|------|-----------
+| ```-filespath files/``` | Path to stored uploads (default is files/)
+| ```-nologs``` | (optionally) disable deletion logs in stdout
+| ```-metapath meta/``` | Path to stored information about uploads (default is meta/)
+

linx-cleanup/linx-cleanup.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/andreimarcu/linx-server/cleanup"
+)
+
+func main() {
+	var filesDir string
+	var metaDir string
+	var noLogs bool
+
+	flag.StringVar(&filesDir, "filespath", "files/",
+		"path to files directory")
+	flag.StringVar(&metaDir, "metapath", "meta/",
+		"path to metadata directory")
+	flag.BoolVar(&noLogs, "nologs", false,
+		"don't log deleted files")
+	flag.Parse()
+
+	cleanup.Cleanup(filesDir, metaDir, noLogs)
+}

server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/andreimarcu/linx-server/backends"
 	"github.com/andreimarcu/linx-server/backends/localfs"
 	"github.com/andreimarcu/linx-server/backends/s3"
+	"github.com/andreimarcu/linx-server/cleanup"
 	"github.com/flosch/pongo2"
 	"github.com/vharitonsky/iniflags"
 	"github.com/zenazn/goji/graceful"
@@ -71,6 +72,7 @@ var Config struct {
 	forceRandomFilename       bool
 	accessKeyCookieExpiry     uint64
 	customPagesDir            string
+	cleanupEveryMinutes       uint64
 }
 
 var Templates = make(map[string]*pongo2.Template)
@@ -150,6 +152,10 @@ func setup() *web.Mux {
 		storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle)
 	} else {
 		storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir)
+		if Config.cleanupEveryMinutes > 0 {
+			go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs)
+		}
+
 	}
 
 	// Template setup
@@ -311,6 +317,8 @@ func main() {
 	flag.Uint64Var(&Config.accessKeyCookieExpiry, "access-cookie-expiry", 0, "Expiration time for access key cookies in seconds (set 0 to use session cookies)")
 	flag.StringVar(&Config.customPagesDir, "custompagespath", "",
 		"path to directory containing .md files to render as custom pages")
+	flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0,
+		"How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)")
 
 	iniflags.Parse()