From 2cac1aaf1eb386b8646842938bda061e57b72941 Mon Sep 17 00:00:00 2001 From: fenos Date: Wed, 7 Dec 2022 09:28:18 +0000 Subject: [PATCH 1/2] chore: whitelist x-client-trace-id header to be logged --- src/monitoring/logger.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/monitoring/logger.ts b/src/monitoring/logger.ts index b2e2abba4..e86f06f56 100644 --- a/src/monitoring/logger.ts +++ b/src/monitoring/logger.ts @@ -67,6 +67,7 @@ const whitelistHeaders = (headers: Record) => { 'x-real-ip', 'x-client-info', 'x-forwarded-user-agent', + 'x-client-trace-id', ] const allowlistedResponseHeaders = [ 'cf-cache-status', From 43f14553933982bc99cb0494c8cc287d71f16600 Mon Sep 17 00:00:00 2001 From: fenos Date: Wed, 7 Dec 2022 20:22:08 +0000 Subject: [PATCH 2/2] feat: rate limiting render routes --- package-lock.json | 196 +++++++++++++++++++- package.json | 2 + src/config.ts | 11 ++ src/http/routes/object/getSignedObject.ts | 6 + src/http/routes/render/index.ts | 7 +- src/http/routes/render/rate-limiter.ts | 41 ++++ src/http/routes/render/renderSignedImage.ts | 7 + src/storage/object.ts | 2 +- src/storage/renderer/renderer.ts | 7 +- src/test/db/docker-compose.yml | 6 + 10 files changed, 276 insertions(+), 9 deletions(-) create mode 100644 src/http/routes/render/rate-limiter.ts diff --git a/package-lock.json b/package-lock.json index 148c3b9f2..7ea70c224 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@aws-sdk/node-http-handler": "^3.178.0", "@aws-sdk/s3-request-presigner": "^3.182.0", "@fastify/multipart": "^7.1.0", + "@fastify/rate-limit": "^7.6.0", "@fastify/swagger": "^7.4.1", "@supabase/postgrest-js": "^0.36.0", "axios": "^0.27.2", @@ -27,6 +28,7 @@ "fastify-plugin": "^4.0.0", "fs-extra": "^10.0.1", "fs-xattr": "^0.3.1", + "ioredis": "^5.2.4", "jsonwebtoken": "^8.5.1", "knex": "^1.0.3", "md5-file": "^5.0.0", @@ -2106,6 +2108,29 @@ "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.1.tgz", "integrity": "sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA==" }, + "node_modules/@fastify/rate-limit": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-7.6.0.tgz", + "integrity": "sha512-XOS7J17dp7d45hMUzBFGJC6B/8znr4/wicvl4gxnbI/upf2ShoOasvLl5J10BFQW/ccqdNURXISb7eHf4Ac4nQ==", + "dependencies": { + "fastify-plugin": "^4.0.0", + "ms": "^2.1.3", + "tiny-lru": "^10.0.0" + } + }, + "node_modules/@fastify/rate-limit/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/@fastify/rate-limit/node_modules/tiny-lru": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.0.1.tgz", + "integrity": "sha512-Vst+6kEsWvb17Zpz14sRJV/f8bUWKhqm6Dc+v08iShmIJ/WxqWytHzCTd6m88pS33rE2zpX34TRmOpAJPloNCA==", + "engines": { + "node": ">=6" + } + }, "node_modules/@fastify/static": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-6.4.0.tgz", @@ -2360,6 +2385,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -4046,6 +4076,14 @@ "node": ">=12" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmake-js": { "version": "6.3.2", "resolved": "https://registry.npmjs.org/cmake-js/-/cmake-js-6.3.2.tgz", @@ -4530,9 +4568,9 @@ } }, "node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, @@ -4616,6 +4654,14 @@ "optional": true, "peer": true }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -5956,6 +6002,29 @@ "node": ">=0.10.0" } }, + "node_modules/ioredis": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", + "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -7085,11 +7154,21 @@ "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -8495,6 +8574,25 @@ "node": ">= 10.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regenerator-runtime": { "version": "0.13.10", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", @@ -8857,6 +8955,11 @@ "node": ">=10" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/stream-browserify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", @@ -11438,6 +11541,28 @@ } } }, + "@fastify/rate-limit": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-7.6.0.tgz", + "integrity": "sha512-XOS7J17dp7d45hMUzBFGJC6B/8znr4/wicvl4gxnbI/upf2ShoOasvLl5J10BFQW/ccqdNURXISb7eHf4Ac4nQ==", + "requires": { + "fastify-plugin": "^4.0.0", + "ms": "^2.1.3", + "tiny-lru": "^10.0.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "tiny-lru": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.0.1.tgz", + "integrity": "sha512-Vst+6kEsWvb17Zpz14sRJV/f8bUWKhqm6Dc+v08iShmIJ/WxqWytHzCTd6m88pS33rE2zpX34TRmOpAJPloNCA==" + } + } + }, "@fastify/static": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-6.4.0.tgz", @@ -11669,6 +11794,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -12978,6 +13108,11 @@ "wrap-ansi": "^7.0.0" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "cmake-js": { "version": "6.3.2", "resolved": "https://registry.npmjs.org/cmake-js/-/cmake-js-6.3.2.tgz", @@ -13391,9 +13526,9 @@ "dev": true }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } @@ -13451,6 +13586,11 @@ "optional": true, "peer": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -14475,6 +14615,22 @@ "optional": true, "peer": true }, + "ioredis": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", + "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -15338,11 +15494,21 @@ "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -16452,6 +16618,19 @@ "resolve": "^1.20.0" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regenerator-runtime": { "version": "0.13.10", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", @@ -16705,6 +16884,11 @@ "escape-string-regexp": "^2.0.0" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "stream-browserify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", diff --git a/package.json b/package.json index ff42d23b9..260fd7a43 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@aws-sdk/node-http-handler": "^3.178.0", "@aws-sdk/s3-request-presigner": "^3.182.0", "@fastify/multipart": "^7.1.0", + "@fastify/rate-limit": "^7.6.0", "@fastify/swagger": "^7.4.1", "@supabase/postgrest-js": "^0.36.0", "axios": "^0.27.2", @@ -42,6 +43,7 @@ "fastify-plugin": "^4.0.0", "fs-extra": "^10.0.1", "fs-xattr": "^0.3.1", + "ioredis": "^5.2.4", "jsonwebtoken": "^8.5.1", "knex": "^1.0.3", "md5-file": "^5.0.0", diff --git a/src/config.ts b/src/config.ts index 8a0820ea8..fe91f535c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -46,6 +46,10 @@ type StorageConfigType = { adminPort: number port: number host: string + enableRateLimiter: boolean + rateLimiterDriver: 'memory' | 'redis' | string + rateLimiterRedisUrl?: string + rateLimiterRenderPathMaxReqSec: number } function getOptionalConfigFromEnv(key: string): string | undefined { @@ -122,5 +126,12 @@ export function getConfig(): StorageConfigType { host: getOptionalConfigFromEnv('HOST') || '0.0.0.0', port: Number(getOptionalConfigFromEnv('PORT')) || 5000, adminPort: Number(getOptionalConfigFromEnv('ADMIN_PORT')) || 5001, + enableRateLimiter: getOptionalConfigFromEnv('ENABLE_RATE_LIMITER') === 'true', + rateLimiterDriver: getOptionalConfigFromEnv('RATE_LIMITER_DRIVER') || 'memory', + rateLimiterRedisUrl: getOptionalConfigFromEnv('RATE_LIMITER_REDIS_URL'), + rateLimiterRenderPathMaxReqSec: parseInt( + getOptionalConfigFromEnv('RATE_LIMITER_RENDER_PATH_MAX_REQ_SEC') || '5', + 10 + ), } } diff --git a/src/http/routes/object/getSignedObject.ts b/src/http/routes/object/getSignedObject.ts index e1ba6f174..ba431a267 100644 --- a/src/http/routes/object/getSignedObject.ts +++ b/src/http/routes/object/getSignedObject.ts @@ -66,6 +66,12 @@ export default async function routes(fastify: FastifyInstance) { } const { url, exp } = payload + const path = `${request.params.bucketName}/${request.params['*']}` + + if (url !== path) { + throw new StorageBackendError('InvalidSignature', 400, 'The url do not match the signature') + } + const s3Key = `${request.tenantId}/${url}` request.log.info(s3Key) diff --git a/src/http/routes/render/index.ts b/src/http/routes/render/index.ts index b2e961dcc..d6cb3687b 100644 --- a/src/http/routes/render/index.ts +++ b/src/http/routes/render/index.ts @@ -4,14 +4,19 @@ import renderAuthenticatedImage from './renderAuthenticatedImage' import renderSignedImage from './renderSignedImage' import { jwt, postgrest, superUserPostgrest, storage, requireTenantFeature } from '../../plugins' import { getConfig } from '../../../config' +import { rateLimiter } from './rate-limiter' -const { enableImageTransformation } = getConfig() +const { enableImageTransformation, enableRateLimiter } = getConfig() export default async function routes(fastify: FastifyInstance) { if (!enableImageTransformation) { return } + if (enableRateLimiter) { + fastify.register(rateLimiter) + } + fastify.register(async function authorizationContext(fastify) { fastify.register(requireTenantFeature('imageTransformation')) fastify.register(jwt) diff --git a/src/http/routes/render/rate-limiter.ts b/src/http/routes/render/rate-limiter.ts new file mode 100644 index 000000000..1b948b8d2 --- /dev/null +++ b/src/http/routes/render/rate-limiter.ts @@ -0,0 +1,41 @@ +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' +import fastifyRateLimit from '@fastify/rate-limit' +import Redis from 'ioredis' +import { getConfig } from '../../../config' + +const { rateLimiterDriver, rateLimiterRedisUrl, rateLimiterRenderPathMaxReqSec } = getConfig() + +export const rateLimiter = fp((fastify: FastifyInstance, ops: any, done: () => void) => { + fastify.register(fastifyRateLimit, { + global: true, + max: rateLimiterRenderPathMaxReqSec * 4, + timeWindow: 4 * 1000, //4s + continueExceeding: true, + nameSpace: 'image-transformation-ratelimit-', + redis: rateLimiterDriver === 'redis' ? new Redis(rateLimiterRedisUrl || '') : undefined, + keyGenerator: function (request) { + const tenant = request.tenantId + const ip = request.headers['x-real-ip'] || request.headers['x-client-ip'] || request.ip + + // exclude query string + const pathWithoutQuery = request.url.split('?').shift() + + return `${tenant}-${ip}-${pathWithoutQuery}` + }, + addHeadersOnExceeding: { + // default show all the response headers when rate limit is not reached + 'x-ratelimit-limit': false, + 'x-ratelimit-remaining': false, + 'x-ratelimit-reset': false, + }, + addHeaders: { + 'x-ratelimit-limit': false, + 'x-ratelimit-remaining': false, + 'x-ratelimit-reset': false, + 'retry-after': false, + }, + }) + + done() +}) diff --git a/src/http/routes/render/renderSignedImage.ts b/src/http/routes/render/renderSignedImage.ts index e82f99558..2d4031c37 100644 --- a/src/http/routes/render/renderSignedImage.ts +++ b/src/http/routes/render/renderSignedImage.ts @@ -58,6 +58,13 @@ export default async function routes(fastify: FastifyInstance) { } const { url, transformations, exp } = payload + + const path = `${request.params.bucketName}/${request.params['*']}` + + if (url !== path) { + throw new StorageBackendError('InvalidSignature', 400, 'The url do not match the signature') + } + const s3Key = `${request.tenantId}/${url}` request.log.info(s3Key) diff --git a/src/storage/object.ts b/src/storage/object.ts index bbf376aa1..7b674c194 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -338,7 +338,7 @@ export class ObjectStorage { 'bucket_id, metadata' ) - await Promise.all([ + await Promise.allSettled([ ObjectRemovedMove.sendWebhook({ tenant: this.db.tenant(), name: sourceObjectName, diff --git a/src/storage/renderer/renderer.ts b/src/storage/renderer/renderer.ts index d0a897cc5..35d385ee7 100644 --- a/src/storage/renderer/renderer.ts +++ b/src/storage/renderer/renderer.ts @@ -43,7 +43,12 @@ export abstract class Renderer { } if (err.$metadata?.httpStatusCode === 404) { - return response.status(404).send() + response.header('cache-control', 'no-store') + return response.status(400).send({ + error: 'Not found', + message: 'The resource was not found', + statusCode: '404', + }) } throw err diff --git a/src/test/db/docker-compose.yml b/src/test/db/docker-compose.yml index 3cd4e4354..39de87ae2 100644 --- a/src/test/db/docker-compose.yml +++ b/src/test/db/docker-compose.yml @@ -34,6 +34,12 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + redis: + image: redis:6.2-alpine + restart: always + ports: + - '6379:6379' + imgproxy: image: darthsim/imgproxy ports: