Remove cached images if they haven't been used in 14 days

.gitignore

@@ -2,10 +2,11 @@ cache
 node_modules
 images.json
 stats.json
+cache.json
 ansible/files/unsplash.key
 ansible/files/unsplash.crt
 ansible/files/unsplash_rsa
 ansible/files/vnstat.db
 ansible/files/photos.zip
 ansible/files/photos.json
-ansible/files/stats.json
+ansible/files/stats.json

buildcache.js

@@ -4,33 +4,43 @@ var async = require('async')
 var config = require('./config')()
 var fs = require('fs')
 
+try {
+  var cache = require(config.cache_metadata_path)
+} catch (error) {
+  var cache = {}
+}
+
 sharp.cache(0)
 
 var imageProcessor = require('./imageProcessor')(sharp, path, config, fs)
 var images = require(config.image_store_path)
 
-fs.mkdir(config.cache_folder_path, function (error) {})
+fs.mkdir(config.cache_folder_path, function (error) {
+  var index = process.argv[2] || 0
+  console.log('Start: %s', index)
 
-var index = process.argv[2] || 0
-console.log('Start: %s', index)
+  if (index > 0) {
+    images.splice(0, index)
+  }
 
-if (index > 0) {
-  images.splice(0, index)
-}
+  async.eachLimit(images, 5, function (image, next) {
+    var width = 458
+    var height = 354
+    var blur = false
+    imageProcessor.getProcessedImage(width, height, null, false, false, image.filename, false, function (error, imagePath) {
+      if (error) {
+        console.log('filePath: ' + image.filename)
+        console.log('imagePath: ' + imagePath)
+        console.log('error: ' + err)
+      }
 
-async.eachLimit(images, 5, function (image, next) {
-  var width = 458
-  var height = 354
-  var blur = false
-  imageProcessor.getProcessedImage(width, height, null, false, false, image.filename, false, function (error, imagePath) {
-    if (error) {
-      console.log('filePath: ' + image.filename)
-      console.log('imagePath: ' + imagePath)
-      console.log('error: ' + err)
-    }
-    console.log('%s done', image.id)
-    next()
+      console.log('%s done', image.id)
+      cache[imagePath] = new Date()
+      next()
+    })
+  }, function (error) {
+    fs.writeFile(config.cache_metadata_path, JSON.stringify(cache), 'utf-8', function (error) {
+      console.log('Done')      
+    })
   })
-}, function (error) {
-  console.log('Done')
 })

config.json

@@ -8,7 +8,8 @@
         "max_height": 5000,
         "max_width": 5000,
         "stats_port": 3000,
-        "port": 5000
+        "port": 5000,
+        "cache_metadata_path": "/opt/unsplash-it/cache.json"
     },
     "development": {
         "folder_path": "../unsplash-downloader/photos",
@@ -19,6 +20,7 @@
         "max_height": 5000,
         "max_width": 5000,
         "stats_port": 3000,
-        "port": 5000
+        "port": 5000,
+        "cache_metadata_path": "./cache.json"
     }
 }

index.js

@@ -9,6 +9,7 @@ if (cluster.isMaster) {
   var io = require('socket.io')(config.stats_port)
   var vnstat = require('vnstat-dumpdb')
   var metadata = require(config.metadata_path)
+  var moment = require('moment')
   console.log('Config:')
   console.log(config)
 
@@ -24,6 +25,12 @@ if (cluster.isMaster) {
     var images = []
   }
 
+  try {
+    var cache = require(config.cache_metadata_path)
+  } catch (e) {
+    var cache = {}
+  }
+
   var publicStats = {}
   var bandWidth = 0
 
@@ -56,26 +63,29 @@ if (cluster.isMaster) {
   fetchBandwidth()
   fetchStats()
 
-  var cleanupAndExit = function () {
-    cleanup()
-    process.exit()
-  }
-
-  var cleanup = function () {
-    saveStatsToFile()
+  var saveToFileAndExit = function () {
+    saveToFile(function () {
+      process.exit(0)
+    })
   }
 
-  var saveStatsToFile = function () {
-    fs.writeFileSync(config.stats_path, JSON.stringify(stats), 'utf8')
+  var saveToFile = function (callback) {
+    fs.writeFile(config.stats_path, JSON.stringify(stats), 'utf8', function (error) {
+      fs.writeFile(config.cache_metadata_path, JSON.stringify(cache), 'utf8', function (error) {
+        if (callback) {
+          callback()
+        }
+      })
+    })
   }
 
-  process.on('exit', cleanup)
-  process.on('SIGINT', cleanupAndExit)
-  process.on('SIGTERM', cleanupAndExit)
+  process.on('exit', saveToFile)
+  process.on('SIGINT', saveToFileAndExit)
+  process.on('SIGTERM', saveToFileAndExit)
   process.on('uncaughtException', function (error) {
     console.log('Uncaught exception: ')
     console.trace(error)
-    cleanupAndExit()
+    saveToFileAndExit()
   })
 
   var loadImages = function () {
@@ -139,30 +149,75 @@ if (cluster.isMaster) {
     images = newImages
 
     fs.writeFile(config.image_store_path, JSON.stringify(newImages), 'utf8', function (error) {
-      startWebServers()
+      findMissingCacheFiles(function () {
+        startWebServers()
+      })
+    })
+  }
+
+  var findMissingCacheFiles = function (callback) {
+    fs.readdir(config.cache_folder_path, function (error, list) {
+      if (error) {
+        console.log('Error reading cache directory!')
+        return callback()
+      }
+
+      async.each(list, function (filename, next) {
+        filename = path.resolve(config.cache_folder_path, filename)
+        if (cache[filename] === undefined) {
+          fs.unlink(filename, function (error) {
+            next()
+          })
+        } else {
+          next()
+        }
+      }, function (error) {
+        callback()
+      })
     })
   }
 
   var startWebServers = function () {
-    fs.mkdir(config.cache_folder_path, function (error) {
-      var cpuCount = require('os').cpus().length - 1
+    var cpuCount = require('os').cpus().length - 1
 
-      if (cpuCount < 2) {
-        cpuCount = 2
-      }
+    if (cpuCount < 2) {
+      cpuCount = 2
+    }
 
-      for (var i = 0, il=cpuCount; i < il; i++) {
-        startWorker()
-      }
+    for (var i = 0, il=cpuCount; i < il; i++) {
+      startWorker()
+    }
+
+    cluster.on('exit', function (worker) {
+      console.log('Worker ' + worker.id + ' died')
+      startWorker()
+    })
+
+    setInterval(function () {
+      saveToFile()
+    }, 1000 * 5)
 
-      cluster.on('exit', function (worker) {
-        console.log('Worker ' + worker.id + ' died')
-        startWorker()
+    var triggerCacheCleanup = function () {
+      cleanupCache(function () {
+        setTimeout(triggerCacheCleanup, 1000 * 60 * 5)
       })
+    }
+
+    setTimeout(triggerCacheCleanup, 1000 * 60 * 5)
+  }
 
-      setInterval(function () {
-        saveStatsToFile()
-      }, 5000)
+  var cleanupCache = function (callback) {
+    async.eachLimit(Object.keys(cache), 100, function (filename, next) {
+      if (moment().diff(cache[filename], 'days') >= 14) {
+        fs.unlink(filename, function (error) {
+          delete cache[filename]
+          next()
+        })
+      } else {
+        next()
+      }
+    }, function (error) {
+      callback()
     })
   }
 
@@ -174,9 +229,12 @@ if (cluster.isMaster) {
 
   var handleWorkerMessage = function (msg) {
     stats.count++
+    cache[msg] = new Date()
   }
 
-  loadImages()
+  fs.mkdir(config.cache_folder_path, function (error) {
+   loadImages() 
+  })
 } else {
   var config = require('./config')()
   require('./server')(function (callback) {

package.json

@@ -12,11 +12,12 @@
   },
   "author": "David Marby <david@dmarby.se> (http://dmarby.se)",
   "dependencies": {
+    "async": "^0.9.0",
     "cors": "^2.4.2",
     "express": "^4.8.2",
+    "moment": "^2.9.0",
     "sharp": "git://github.com/lovell/sharp.git#edge",
     "socket.io": "^1.0.6",
-    "vnstat-dumpdb": "^1.0.2",
-    "async": "^0.9.0"
+    "vnstat-dumpdb": "^1.0.2"
   }
 }

server.js

@@ -113,7 +113,7 @@ module.exports = function (callback) {
           }
 
           res.sendFile(imagePath)
-          countImage()
+          process.send(imagePath)
         })
       })
     })
@@ -156,9 +156,5 @@ module.exports = function (callback) {
     res.send({ error: message })
   }
 
-  var countImage = function () {
-    process.send('count')
-  }
-
   callback(app)
 }