Skip to content

fix: add X-Robots-Tag header to prevent robot indexing and following #776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix: add X-Robots-Tag header to prevent robot indexing and following #776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9f52aec
fix: add X-Robots-Tag header to prevent robot indexing and following
itslenny Oct 6, 2025
98440df
add ability to override x-robots-tag header
itslenny Oct 7, 2025
a019e9b
add x-robots-tag validation
itslenny Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/http/routes/object/createObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ interface createObjectRequestInterface extends RequestGenericInterface {
'content-type': string
'cache-control'?: string
'x-upsert'?: string
'x-robots-tag'?: string
}
}

Expand Down
9 changes: 7 additions & 2 deletions src/http/routes/object/getObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,21 @@ async function requestHandler(

if (bucket.public) {
// request is authenticated but we still use the superUser as we don't need to check RLS
obj = await request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id, version')
obj = await request.storage
.asSuperUser()
.from(bucketName)
.findObject(objectName, 'id, version, metadata')
} else {
// request is authenticated use RLS
obj = await request.storage.from(bucketName).findObject(objectName, 'id, version')
obj = await request.storage.from(bucketName).findObject(objectName, 'id, version, metadata')
}

return request.storage.renderer('asset').render(request, response, {
bucket: storageS3Bucket,
key: s3Key,
version: obj.version,
download,
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand All @@ -95,6 +99,7 @@ export default async function routes(fastify: FastifyInstance) {
// @todo add success response schema here
schema: {
params: getObjectParamsSchema,
querystring: getObjectQuerySchema,
headers: { $ref: 'authSchema#' },
summary,
response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } },
Expand Down
7 changes: 6 additions & 1 deletion src/http/routes/object/getPublicObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export default async function routes(fastify: FastifyInstance) {
exposeHeadRoute: false,
schema: {
params: getPublicObjectParamsSchema,
querystring: getObjectQuerySchema,
summary,
response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } },
tags: ['object'],
Expand All @@ -55,7 +56,10 @@ export default async function routes(fastify: FastifyInstance) {
request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
isPublic: true,
}),
request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
request.storage
.asSuperUser()
.from(bucketName)
.findObject(objectName, 'id,version,metadata'),
])

// send the object from s3
Expand All @@ -70,6 +74,7 @@ export default async function routes(fastify: FastifyInstance) {
key: s3Key,
version: obj.version,
download,
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand Down
3 changes: 2 additions & 1 deletion src/http/routes/object/getSignedObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,15 @@ export default async function routes(fastify: FastifyInstance) {
const obj = await request.storage
.asSuperUser()
.from(bucketName)
.findObject(objParts.join('/'), 'id,version')
.findObject(objParts.join('/'), 'id,version,metadata')

return request.storage.renderer('asset').render(request, response, {
bucket: storageS3Bucket,
key: s3Key,
version: obj.version,
download,
expires: new Date(exp * 1000).toUTCString(),
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand Down
1 change: 1 addition & 0 deletions src/http/routes/object/updateObject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ interface updateObjectRequestInterface extends RequestGenericInterface {
'content-type': string
'cache-control'?: string
'x-upsert'?: string
'x-robots-tag'?: string
}
}

Expand Down
5 changes: 4 additions & 1 deletion src/http/routes/render/renderAuthenticatedImage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ export default async function routes(fastify: FastifyInstance) {
const { bucketName } = request.params
const objectName = request.params['*']

const obj = await request.storage.from(bucketName).findObject(objectName, 'id,version')
const obj = await request.storage
.from(bucketName)
.findObject(objectName, 'id,version,metadata')

const s3Key = request.storage.location.getKeyLocation({
tenantId: request.tenantId,
Expand All @@ -73,6 +75,7 @@ export default async function routes(fastify: FastifyInstance) {
key: s3Key,
version: obj.version,
download,
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand Down
6 changes: 5 additions & 1 deletion src/http/routes/render/renderPublicImage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ export default async function routes(fastify: FastifyInstance) {
request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
isPublic: true,
}),
request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
request.storage
.asSuperUser()
.from(bucketName)
.findObject(objectName, 'id,version,metadata'),
])

const s3Key = `${request.tenantId}/${bucketName}/${objectName}`
Expand All @@ -74,6 +77,7 @@ export default async function routes(fastify: FastifyInstance) {
key: s3Key,
version: obj.version,
download,
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand Down
3 changes: 2 additions & 1 deletion src/http/routes/render/renderSignedImage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export default async function routes(fastify: FastifyInstance) {
const obj = await request.storage
.asSuperUser()
.from(bucketName)
.findObject(objParts.join('/'), 'id,version')
.findObject(objParts.join('/'), 'id,version,metadata')

const renderer = request.storage.renderer('image') as ImageRenderer

Expand All @@ -102,6 +102,7 @@ export default async function routes(fastify: FastifyInstance) {
version: obj.version,
download,
expires: new Date(exp * 1000).toUTCString(),
xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
signal: request.signals.disconnect.signal,
})
}
Expand Down
8 changes: 8 additions & 0 deletions src/internal/errors/codes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,14 @@ export const ERRORS = {
message: `mime type ${mimeType} is not supported`,
}),

InvalidXRobotsTag: (message: string) =>
new StorageBackendError({
error: 'invalid_x_robots_tag',
code: ErrorCode.InvalidRequest,
httpStatusCode: 400,
message: `Invalid X-Robots-Tag header: ${message}`,
}),

InvalidRange: () =>
new StorageBackendError({
error: 'invalid_range',
Expand Down
1 change: 1 addition & 0 deletions src/storage/backend/adapter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export type ObjectMetadata = {
eTag: string
contentRange?: string
httpStatusCode?: number
xRobotsTag?: string
}

export type UploadPart = {
Expand Down
12 changes: 12 additions & 0 deletions src/storage/renderer/renderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ import { ObjectMetadata } from '../backend'
import { Readable } from 'stream'
import { getConfig } from '../../config'
import { Obj } from '../schemas'
import { validateXRobotsTag } from '@storage/validators/x-robots-tag'

export interface RenderOptions {
bucket: string
key: string
version: string | undefined
download?: string
expires?: string
xRobotsTag?: string
object?: Obj
signal?: AbortSignal
}
Expand Down Expand Up @@ -73,13 +75,23 @@ export abstract class Renderer {
data: AssetResponse,
options: RenderOptions
) {
let xRobotsTag = 'none'
if (options.xRobotsTag) {
try {
// allow overriding x-robots-tag header only with valid values
validateXRobotsTag(options.xRobotsTag)
xRobotsTag = options.xRobotsTag
} catch {}
}

response
.status(data.metadata.httpStatusCode ?? 200)
.header('Accept-Ranges', 'bytes')
.header('Content-Type', normalizeContentType(data.metadata.mimetype))
.header('ETag', data.metadata.eTag)
.header('Content-Length', data.metadata.contentLength)
.header('Last-Modified', data.metadata.lastModified?.toUTCString())
.header('X-Robots-Tag', xRobotsTag)

if (options.expires) {
response.header('Expires', options.expires)
Expand Down
20 changes: 16 additions & 4 deletions src/storage/uploader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { getConfig } from '../config'
import { logger, logSchema } from '@internal/monitoring'
import { Readable } from 'stream'
import { StorageObjectLocator } from '@storage/locator'
import { validateXRobotsTag } from './validators/x-robots-tag'

const { storageS3Bucket, uploadFileSizeLimitStandard } = getConfig()

Expand All @@ -20,7 +21,8 @@ interface FileUpload {
mimeType: string
cacheControl: string
isTruncated: () => boolean
userMetadata?: Record<string, any>
xRobotsTag?: string
userMetadata?: Record<string, unknown>
}

export interface UploadRequest {
Expand Down Expand Up @@ -112,6 +114,10 @@ export class Uploader {
request.signal
)

if (request.file.xRobotsTag) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should start adding strict validation for these headers.
Let's start adding the validation for the x-robot-tags

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added validation to where the values are stored via API and before applying the header to our responses. So, we won't serve a file with an invalid header in the case an invalid value was set in the DB directly (bypassing the API check).

objectMetadata.xRobotsTag = request.file.xRobotsTag
}

if (file.isTruncated()) {
throw ERRORS.EntityTooLarge()
}
Expand Down Expand Up @@ -301,9 +307,14 @@ export async function fileUploadFromRequest(
}
): Promise<FileUpload & { maxFileSize: number }> {
const contentType = request.headers['content-type']
const xRobotsTag = request.headers['x-robots-tag'] as string | undefined

if (xRobotsTag) {
validateXRobotsTag(xRobotsTag)
}

let body: Readable
let userMetadata: Record<string, any> | undefined
let userMetadata: Record<string, unknown> | undefined
let mimeType: string
let isTruncated: () => boolean
let maxFileSize = 0
Expand Down Expand Up @@ -349,7 +360,7 @@ export async function fileUploadFromRequest(

try {
userMetadata = JSON.parse(customMd)
} catch (e) {
} catch {
// no-op
}
}
Expand Down Expand Up @@ -388,14 +399,15 @@ export async function fileUploadFromRequest(
isTruncated,
userMetadata,
maxFileSize,
xRobotsTag,
}
}

export function parseUserMetadata(metadata: string) {
try {
const json = Buffer.from(metadata, 'base64').toString('utf8')
return JSON.parse(json) as Record<string, string>
} catch (e) {
} catch {
// no-op
return undefined
}
Expand Down
Loading
Loading