Permalink
Browse files

Remove cached images if they haven't been used in 14 days

  • Loading branch information...
1 parent 6a14a98 commit 1c83e88bda915b2af75f3e4b0954fd1c9b1dfd07 @DMarby committed Apr 3, 2015
Showing with 127 additions and 59 deletions.
  1. +2 −1 .gitignore
  2. +30 −20 buildcache.js
  3. +4 −2 config.json
  4. +87 −29 index.js
  5. +3 −2 package.json
  6. +1 −5 server.js
View
@@ -2,10 +2,11 @@ cache
node_modules
images.json
stats.json
+cache.json
ansible/files/unsplash.key
ansible/files/unsplash.crt
ansible/files/unsplash_rsa
ansible/files/vnstat.db
ansible/files/photos.zip
ansible/files/photos.json
-ansible/files/stats.json
+ansible/files/stats.json
View
@@ -4,33 +4,43 @@ var async = require('async')
var config = require('./config')()
var fs = require('fs')
+try {
+ var cache = require(config.cache_metadata_path)
+} catch (error) {
+ var cache = {}
+}
+
sharp.cache(0)
var imageProcessor = require('./imageProcessor')(sharp, path, config, fs)
var images = require(config.image_store_path)
-fs.mkdir(config.cache_folder_path, function (error) {})
+fs.mkdir(config.cache_folder_path, function (error) {
+ var index = process.argv[2] || 0
+ console.log('Start: %s', index)
-var index = process.argv[2] || 0
-console.log('Start: %s', index)
+ if (index > 0) {
+ images.splice(0, index)
+ }
-if (index > 0) {
- images.splice(0, index)
-}
+ async.eachLimit(images, 5, function (image, next) {
+ var width = 458
+ var height = 354
+ var blur = false
+ imageProcessor.getProcessedImage(width, height, null, false, false, image.filename, false, function (error, imagePath) {
+ if (error) {
+ console.log('filePath: ' + image.filename)
+ console.log('imagePath: ' + imagePath)
+ console.log('error: ' + err)
+ }
-async.eachLimit(images, 5, function (image, next) {
- var width = 458
- var height = 354
- var blur = false
- imageProcessor.getProcessedImage(width, height, null, false, false, image.filename, false, function (error, imagePath) {
- if (error) {
- console.log('filePath: ' + image.filename)
- console.log('imagePath: ' + imagePath)
- console.log('error: ' + err)
- }
- console.log('%s done', image.id)
- next()
+ console.log('%s done', image.id)
+ cache[imagePath] = new Date()
+ next()
+ })
+ }, function (error) {
+ fs.writeFile(config.cache_metadata_path, JSON.stringify(cache), 'utf-8', function (error) {
+ console.log('Done')
+ })
})
-}, function (error) {
- console.log('Done')
})
View
@@ -8,7 +8,8 @@
"max_height": 5000,
"max_width": 5000,
"stats_port": 3000,
- "port": 5000
+ "port": 5000,
+ "cache_metadata_path": "/opt/unsplash-it/cache.json"
},
"development": {
"folder_path": "../unsplash-downloader/photos",
@@ -19,6 +20,7 @@
"max_height": 5000,
"max_width": 5000,
"stats_port": 3000,
- "port": 5000
+ "port": 5000,
+ "cache_metadata_path": "./cache.json"
}
}
View
@@ -9,6 +9,7 @@ if (cluster.isMaster) {
var io = require('socket.io')(config.stats_port)
var vnstat = require('vnstat-dumpdb')
var metadata = require(config.metadata_path)
+ var moment = require('moment')
console.log('Config:')
console.log(config)
@@ -24,6 +25,12 @@ if (cluster.isMaster) {
var images = []
}
+ try {
+ var cache = require(config.cache_metadata_path)
+ } catch (e) {
+ var cache = {}
+ }
+
var publicStats = {}
var bandWidth = 0
@@ -56,26 +63,29 @@ if (cluster.isMaster) {
fetchBandwidth()
fetchStats()
- var cleanupAndExit = function () {
- cleanup()
- process.exit()
- }
-
- var cleanup = function () {
- saveStatsToFile()
+ var saveToFileAndExit = function () {
+ saveToFile(function () {
+ process.exit(0)
+ })
}
- var saveStatsToFile = function () {
- fs.writeFileSync(config.stats_path, JSON.stringify(stats), 'utf8')
+ var saveToFile = function (callback) {
+ fs.writeFile(config.stats_path, JSON.stringify(stats), 'utf8', function (error) {
+ fs.writeFile(config.cache_metadata_path, JSON.stringify(cache), 'utf8', function (error) {
+ if (callback) {
+ callback()
+ }
+ })
+ })
}
- process.on('exit', cleanup)
- process.on('SIGINT', cleanupAndExit)
- process.on('SIGTERM', cleanupAndExit)
+ process.on('exit', saveToFile)
+ process.on('SIGINT', saveToFileAndExit)
+ process.on('SIGTERM', saveToFileAndExit)
process.on('uncaughtException', function (error) {
console.log('Uncaught exception: ')
console.trace(error)
- cleanupAndExit()
+ saveToFileAndExit()
})
var loadImages = function () {
@@ -139,30 +149,75 @@ if (cluster.isMaster) {
images = newImages
fs.writeFile(config.image_store_path, JSON.stringify(newImages), 'utf8', function (error) {
- startWebServers()
+ findMissingCacheFiles(function () {
+ startWebServers()
+ })
+ })
+ }
+
+ var findMissingCacheFiles = function (callback) {
+ fs.readdir(config.cache_folder_path, function (error, list) {
+ if (error) {
+ console.log('Error reading cache directory!')
+ return callback()
+ }
+
+ async.each(list, function (filename, next) {
+ filename = path.resolve(config.cache_folder_path, filename)
+ if (cache[filename] === undefined) {
+ fs.unlink(filename, function (error) {
+ next()
+ })
+ } else {
+ next()
+ }
+ }, function (error) {
+ callback()
+ })
})
}
var startWebServers = function () {
- fs.mkdir(config.cache_folder_path, function (error) {
- var cpuCount = require('os').cpus().length - 1
+ var cpuCount = require('os').cpus().length - 1
- if (cpuCount < 2) {
- cpuCount = 2
- }
+ if (cpuCount < 2) {
+ cpuCount = 2
+ }
- for (var i = 0, il=cpuCount; i < il; i++) {
- startWorker()
- }
+ for (var i = 0, il=cpuCount; i < il; i++) {
+ startWorker()
+ }
+
+ cluster.on('exit', function (worker) {
+ console.log('Worker ' + worker.id + ' died')
+ startWorker()
+ })
+
+ setInterval(function () {
+ saveToFile()
+ }, 1000 * 5)
- cluster.on('exit', function (worker) {
- console.log('Worker ' + worker.id + ' died')
- startWorker()
+ var triggerCacheCleanup = function () {
+ cleanupCache(function () {
+ setTimeout(triggerCacheCleanup, 1000 * 60 * 5)
})
+ }
+
+ setTimeout(triggerCacheCleanup, 1000 * 60 * 5)
+ }
- setInterval(function () {
- saveStatsToFile()
- }, 5000)
+ var cleanupCache = function (callback) {
+ async.eachLimit(Object.keys(cache), 100, function (filename, next) {
+ if (moment().diff(cache[filename], 'days') >= 14) {
+ fs.unlink(filename, function (error) {
+ delete cache[filename]
+ next()
+ })
+ } else {
+ next()
+ }
+ }, function (error) {
+ callback()
})
}
@@ -174,9 +229,12 @@ if (cluster.isMaster) {
var handleWorkerMessage = function (msg) {
stats.count++
+ cache[msg] = new Date()
}
- loadImages()
+ fs.mkdir(config.cache_folder_path, function (error) {
+ loadImages()
+ })
} else {
var config = require('./config')()
require('./server')(function (callback) {
View
@@ -12,11 +12,12 @@
},
"author": "David Marby <david@dmarby.se> (http://dmarby.se)",
"dependencies": {
+ "async": "^0.9.0",
"cors": "^2.4.2",
"express": "^4.8.2",
+ "moment": "^2.9.0",
"sharp": "git://github.com/lovell/sharp.git#edge",
"socket.io": "^1.0.6",
- "vnstat-dumpdb": "^1.0.2",
- "async": "^0.9.0"
+ "vnstat-dumpdb": "^1.0.2"
}
}
View
@@ -113,7 +113,7 @@ module.exports = function (callback) {
}
res.sendFile(imagePath)
- countImage()
+ process.send(imagePath)
})
})
})
@@ -156,9 +156,5 @@ module.exports = function (callback) {
res.send({ error: message })
}
- var countImage = function () {
- process.send('count')
- }
-
callback(app)
}

0 comments on commit 1c83e88

Please sign in to comment.