From 0fa1def556106eed2ccfa1f66d5085fcbe7f7a05 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 4 Apr 2025 18:09:37 -0400 Subject: [PATCH 1/2] add support for hidden collections parameter for applying auth --- CHANGELOG.md | 6 + README.md | 33 +++++ package.json | 2 +- src/lambdas/api/app.js | 65 +++++----- src/lib/api.js | 114 ++++++++++++------ src/lib/database.js | 4 +- src/lib/errors.js | 10 +- src/lib/s3-utils.ts | 2 +- tests/system/test-api-collection-items-get.js | 21 ++++ tests/system/test-api-get-aggregate.js | 88 ++++++++++++++ .../test-api-get-collection-aggregate.js | 21 ++++ .../test-api-get-collection-aggregations.js | 21 ++++ .../test-api-get-collection-queryables.js | 21 ++++ tests/system/test-api-get-collection.js | 19 +++ tests/system/test-api-get-collections.js | 22 +++- tests/system/test-api-item-get.js | 56 ++++++++- tests/system/test-api-search-get.js | 37 ++++++ tests/system/test-api-search-post.js | 42 +++++++ 18 files changed, 509 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9721a854..d0d2f96f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use Node 22 by default. +## Added + +- To all endpoints that depend on collections, add support for a query parameter (GET) + or body field (POST) `_collections` that will filter to only those collections, but + will not reveal that in link contents. + ## [3.11.0] - 2025-03-27 ### Added diff --git a/README.md b/README.md index e02c65f3..98a4a1ed 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ - [Warnings](#warnings) - [4.0.0](#400) - [Node 22 update](#node-22-update) + - [Hidden collections filter](#hidden-collections-filter) - [3.10.0](#3100) - [Node 20 update](#node-20-update) - [3.1.0](#310) @@ -49,6 +50,7 @@ - [Filter Extension](#filter-extension) - [Query Extension](#query-extension) - [Aggregation](#aggregation) + - [Hidden collections filter for authorization](#hidden-collections-filter-for-authorization) - [Ingesting Data](#ingesting-data) - [Ingesting large items](#ingesting-large-items) - [Subscribing to SNS Topics](#subscribing-to-sns-topics) @@ -175,6 +177,13 @@ The default Lambda deployment environment is now Node 22. To update the deployment to use Node 22, modify the serverless config file value `provider.runtime` to be `nodejs22.x` and the application re-deployed. +#### Hidden collections filter + +To all endpoints that depend on collections, there is now support for a query parameter +(GET) or body field (POST) `_collections` that will filter to only those collections, but +will not reveal that in link contents. This is useful for the application of permissions +to only certain collections. + ### 3.10.0 #### Node 20 update @@ -1084,6 +1093,30 @@ Available aggregations are: - geometry_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.geometry) - geometry_geotile_grid_frequency ([geotile grid](https://opensearch.org/docs/latest/aggregations/bucket/geotile-grid/) on Item.geometry) +## Hidden collections filter for authorization + +All endpoints that involve the use of Collections support the use of a "hidden" query +parameter named (for GET requests) or body JSON field (for POST requests) named +`_collections` that can be used by an authorization proxy (e.g., a pre-hook Lambda) +to filter the collections a user has access to. This parameter/field will be excluded +from pagination links, so it does not need to be removed on egress. + +The endpoints this applies to are: + +- /collections +- /collections/:collectionId +- /collections/:collectionId/queryables +- /collections/:collectionId/aggregations +- /collections/:collectionId/aggregate +- /collections/:collectionId/items +- /collections/:collectionId/items/:itemId +- /collections/:collectionId/items/:itemId/thumbnail +- /search +- /aggregate + +The five endpoints of the Transaction Extension do not use this parameter, as there are +other authorization considerations for these, that are left as future work. + ## Ingesting Data STAC Collections and Items are ingested by the `ingest` Lambda function, however this Lambda is not invoked directly by a user, it consumes records from the `stac-server--queue` SQS. To add STAC Items or Collections to the queue, publish them to the SNS Topic `stac-server--ingest`. diff --git a/package.json b/package.json index b23d067c..0b912cc3 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "deploy": "sls deploy", "sls-remove": "sls remove", "package": "sls package", - "serve": "REQUEST_LOGGING_FORMAT=dev LOG_LEVEL=debug STAC_API_URL=http://localhost:3000 ENABLE_TRANSACTIONS_EXTENSION=true nodemon --esm ./src/lambdas/api/local.ts", + "serve": "REQUEST_LOGGING_FORMAT=dev LOG_LEVEL=debug STAC_API_URL=http://localhost:3000 ENABLE_TRANSACTIONS_EXTENSION=true nodemon --exec node --loader ts-node/esm ./src/lambdas/api/local.ts", "build-api-docs": "npx @redocly/cli build-docs src/lambdas/api/openapi.yaml -o ./docs/index.html", "prepare": "husky" }, diff --git a/src/lambdas/api/app.js b/src/lambdas/api/app.js index 3650d382..bd4306f0 100644 --- a/src/lambdas/api/app.js +++ b/src/lambdas/api/app.js @@ -6,7 +6,7 @@ import path from 'path' import { fileURLToPath } from 'url' import database from '../../lib/database.js' import api from '../../lib/api.js' -import { ValidationError } from '../../lib/errors.js' +import { NotFoundError, ValidationError } from '../../lib/errors.js' import { readFile } from '../../lib/fs.js' import addEndpoint from './middleware/add-endpoint.js' import logger from '../../lib/logger.js' @@ -153,7 +153,7 @@ app.get('/aggregations', async (req, res, next) => { app.get('/collections', async (req, res, next) => { try { - const response = await api.getCollections(database, req.endpoint) + const response = await api.getCollections(database, req.endpoint, req.query) if (response instanceof Error) next(createError(500, response.message)) else res.json(response) } catch (error) { @@ -185,7 +185,7 @@ app.post('/collections', async (req, res, next) => { app.get('/collections/:collectionId', async (req, res, next) => { const { collectionId } = req.params try { - const response = await api.getCollection(collectionId, database, req.endpoint) + const response = await api.getCollection(collectionId, database, req.endpoint, req.query) if (response instanceof Error) next(createError(404)) else res.json(response) } catch (error) { @@ -196,7 +196,9 @@ app.get('/collections/:collectionId', async (req, res, next) => { app.get('/collections/:collectionId/queryables', async (req, res, next) => { const { collectionId } = req.params try { - const queryables = await api.getCollectionQueryables(collectionId, database, req.endpoint) + const queryables = await api.getCollectionQueryables( + collectionId, database, req.endpoint, req.query + ) if (queryables instanceof Error) next(createError(404)) else { @@ -215,7 +217,9 @@ app.get('/collections/:collectionId/queryables', async (req, res, next) => { app.get('/collections/:collectionId/aggregations', async (req, res, next) => { const { collectionId } = req.params try { - const aggs = await api.getCollectionAggregations(collectionId, database, req.endpoint) + const aggs = await api.getCollectionAggregations( + collectionId, database, req.endpoint, req.query + ) if (aggs instanceof Error) next(createError(404)) else res.json(aggs) } catch (error) { @@ -231,7 +235,9 @@ app.get('/collections/:collectionId/aggregate', async (req, res, next) => { const { collectionId } = req.params try { - const response = await api.getCollection(collectionId, database, req.endpoint) + const response = await api.getCollection( + collectionId, database, req.endpoint, req.query + ) if (response instanceof Error) next(createError(404)) else { @@ -249,20 +255,22 @@ app.get('/collections/:collectionId/aggregate', app.get('/collections/:collectionId/items', async (req, res, next) => { const { collectionId } = req.params try { - const response = await api.getCollection(collectionId, database, req.endpoint) + if ((await api.getCollection( + collectionId, database, req.endpoint, req.query + )) instanceof Error) { + next(createError(404)) + } - if (response instanceof Error) next(createError(404)) - else { - const items = await api.searchItems( + res.type('application/geo+json') + res.json( + await api.searchItems( collectionId, req.query, database, req.endpoint, 'GET' ) - res.type('application/geo+json') - res.json(items) - } + ) } catch (error) { if (error instanceof ValidationError) { next(createError(400, error.message)) @@ -280,7 +288,7 @@ app.post('/collections/:collectionId/items', async (req, res, next) => { if (req.body.collection && req.body.collection !== collectionId) { next(createError(400, 'Collection resource URI must match collection in body')) } else { - const collectionRes = await api.getCollection(collectionId, database, req.endpoint) + const collectionRes = await api.getCollection(collectionId, database, req.endpoint, req.query) if (collectionRes instanceof Error) next(createError(404)) else { try { @@ -312,15 +320,14 @@ app.get('/collections/:collectionId/items/:itemId', async (req, res, next) => { collectionId, itemId, database, - req.endpoint + req.endpoint, + req.query ) - if (response instanceof Error) { - if (response.message === 'Item not found') { - next(createError(404)) - } else { - next(createError(500)) - } + if (response instanceof NotFoundError) { + next(createError(404)) + } else if (response instanceof Error) { + next(createError(500)) } else { res.type('application/geo+json') res.json(response) @@ -339,7 +346,7 @@ app.put('/collections/:collectionId/items/:itemId', async (req, res, next) => { } else if (req.body.id && req.body.id !== itemId) { next(createError(400, 'Item ID in resource URI must match id in body')) } else { - const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint) + const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint, req.query) if (itemRes instanceof Error) next(createError(404)) else { req.body.collection = collectionId @@ -372,7 +379,7 @@ app.patch('/collections/:collectionId/items/:itemId', async (req, res, next) => } else if (req.body.id && req.body.id !== itemId) { next(createError(400, 'Item ID in resource URI must match id in body')) } else { - const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint) + const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint, req.query) if (itemRes instanceof Error) next(createError(404)) else { try { @@ -418,15 +425,13 @@ app.get('/collections/:collectionId/items/:itemId/thumbnail', async (req, res, n collectionId, itemId, database, + req.query ) - if (response instanceof Error) { - if (response.message === 'Item not found' - || response.message === 'Thumbnail not found') { - next(createError(404)) - } else { - next(createError(500)) - } + if (response instanceof NotFoundError) { + next(createError(404)) + } else if (response instanceof Error) { + next(createError(500)) } else { res.redirect(response.location) } diff --git a/src/lib/api.js b/src/lib/api.js index f08fd838..6baa47e3 100644 --- a/src/lib/api.js +++ b/src/lib/api.js @@ -2,7 +2,7 @@ import { pickBy, assign, get as getNested } from 'lodash-es' import { DateTime } from 'luxon' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3' -import { ValidationError } from './errors.js' +import { NotFoundError, ValidationError } from './errors.js' import { isIndexNotFoundError } from './database.js' import logger from './logger.js' import { bboxToPolygon } from './geo-utils.js' @@ -315,9 +315,8 @@ const extractFields = function (params) { return fieldRules } -const extractIds = function (params) { +const parseIds = function (ids) { let idsRules - const { ids } = params if (ids) { if (typeof ids === 'string') { try { @@ -332,21 +331,16 @@ const extractIds = function (params) { return idsRules } +const extractIds = function (params) { + return parseIds(params.ids) +} + +const extractAllowedCollectionIds = function (params) { + return parseIds(params._collections) +} + const extractCollectionIds = function (params) { - let idsRules - const { collections } = params - if (collections) { - if (typeof collections === 'string') { - try { - idsRules = JSON.parse(collections) - } catch (_) { - idsRules = collections.split(',') - } - } else { - idsRules = collections.slice() - } - } - return idsRules + return parseIds(params.collections) } export const parsePath = function (inpath) { @@ -515,7 +509,7 @@ const wrapResponseInFeatureCollection = function ( } } -const buildPaginationLinks = function (limit, parameters, bbox, intersects, endpoint, +const buildPaginationLinks = function (limit, parameters, bbox, intersects, collections, endpoint, httpMethod, sortby, items) { if (items.length) { const dictToURI = (dict) => ( @@ -533,7 +527,7 @@ const buildPaginationLinks = function (limit, parameters, bbox, intersects, endp } } value = sortFields.join(',') - } else if (p === 'collections') { + } else if (p === 'collections') { // TODO value = value.toString() } else { value = JSON.stringify(value) @@ -559,7 +553,7 @@ const buildPaginationLinks = function (limit, parameters, bbox, intersects, endp method: httpMethod, type: 'application/geo+json' } - const nextParams = pickBy(assign(parameters, { bbox, intersects, limit, next })) + const nextParams = pickBy(assign(parameters, { bbox, intersects, limit, next, collections })) if (httpMethod === 'GET') { const nextQueryParameters = dictToURI(nextParams) link.href = `${endpoint}?${nextQueryParameters}` @@ -594,7 +588,11 @@ const searchItems = async function (collectionId, queryParameters, backend, endp const filter = extractCql2Filter(queryParameters) const fields = extractFields(queryParameters) const ids = extractIds(queryParameters) - const collections = extractCollectionIds(queryParameters) + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + const specifiedCollectionIds = extractCollectionIds(queryParameters) + const collections = allowedCollectionIds ? allowedCollectionIds.filter( + (x) => !specifiedCollectionIds || specifiedCollectionIds.includes(x) + ) : specifiedCollectionIds const limit = extractLimit(queryParameters) const page = extractPage(queryParameters) @@ -656,7 +654,15 @@ const searchItems = async function (collectionId, queryParameters, backend, endp const { results: responseItems, context } = esResponse const paginationLinks = buildPaginationLinks( - limit, searchParams, bbox, intersects, newEndpoint, httpMethod, sortby, responseItems + limit, + searchParams, + bbox, + intersects, + specifiedCollectionIds, + newEndpoint, + httpMethod, + sortby, + responseItems ) // @ts-ignore @@ -724,7 +730,11 @@ const aggregate = async function ( const query = extractStacQuery(queryParameters) const filter = extractCql2Filter(queryParameters) const ids = extractIds(queryParameters) - const collections = extractCollectionIds(queryParameters) + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + const specifiedCollectionIds = extractCollectionIds(queryParameters) + const collections = allowedCollectionIds ? allowedCollectionIds.filter( + (x) => !specifiedCollectionIds || specifiedCollectionIds.includes(x) + ) : specifiedCollectionIds const searchParams = pickBy({ datetime, @@ -996,7 +1006,12 @@ const validateAdditionalProperties = (queryables) => { } } -const getCollectionQueryables = async (collectionId, backend, endpoint = '') => { +const getCollectionQueryables = async (collectionId, backend, endpoint, queryParameters) => { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + if (allowedCollectionIds && !allowedCollectionIds.includes(collectionId)) { + return new NotFoundError() + } + const collection = await backend.getCollection(collectionId) if (collection instanceof Error) { @@ -1009,7 +1024,12 @@ const getCollectionQueryables = async (collectionId, backend, endpoint = '') => return queryables } -const getCollectionAggregations = async (collectionId, backend, endpoint = '') => { +const getCollectionAggregations = async (collectionId, backend, endpoint, queryParameters) => { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + if (allowedCollectionIds && !allowedCollectionIds.includes(collectionId)) { + return new NotFoundError() + } + const collection = await backend.getCollection(collectionId) if (collection instanceof Error) { @@ -1146,7 +1166,7 @@ const deleteUnusedFields = (collection) => { delete collection.aggregations } -const getCollections = async function (backend, endpoint = '') { +const getCollections = async function (backend, endpoint, queryParameters) { // TODO: implement proper pagination, as this will only return up to // COLLECTION_LIMIT collections const collectionsOrError = await backend.getCollections(1, COLLECTION_LIMIT) @@ -1154,13 +1174,18 @@ const getCollections = async function (backend, endpoint = '') { return collectionsOrError } - for (const collection of collectionsOrError) { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + const collections = collectionsOrError.filter( + (c) => !allowedCollectionIds || allowedCollectionIds.includes(c.id) + ) + + for (const collection of collections) { deleteUnusedFields(collection) } - const linkedCollections = addCollectionLinks(collectionsOrError, endpoint) + const linkedCollections = addCollectionLinks(collections, endpoint) const resp = { - collections: collectionsOrError, + collections, links: [ { rel: 'self', @@ -1183,10 +1208,15 @@ const getCollections = async function (backend, endpoint = '') { return resp } -const getCollection = async function (collectionId, backend, endpoint = '') { +const getCollection = async function (collectionId, backend, endpoint, queryParameters) { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + if (allowedCollectionIds && !allowedCollectionIds.includes(collectionId)) { + return new NotFoundError() + } + const result = await backend.getCollection(collectionId) if (result instanceof Error) { - return new Error('Collection not found') + return new NotFoundError() } deleteUnusedFields(result) @@ -1208,14 +1238,19 @@ const createCollection = async function (collection, backend) { return new Error(`Error creating collection ${collection}`) } -const getItem = async function (collectionId, itemId, backend, endpoint = '') { +const getItem = async function (collectionId, itemId, backend, endpoint, queryParameters) { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + if (allowedCollectionIds && !allowedCollectionIds.includes(collectionId)) { + return new NotFoundError() + } + const itemQuery = { collections: [collectionId], id: itemId } const { results } = await backend.search(itemQuery) const [it] = addItemLinks(results, endpoint) if (it) { return it } - return new Error('Item not found') + return new NotFoundError() } const partialUpdateItem = async function ( @@ -1258,12 +1293,17 @@ const deleteItem = async function (collectionId, itemId, backend) { return new Error(`Error deleting item ${collectionId}/${itemId}`) } -const getItemThumbnail = async function (collectionId, itemId, backend) { +const getItemThumbnail = async function (collectionId, itemId, backend, queryParameters) { + const allowedCollectionIds = extractAllowedCollectionIds(queryParameters) + if (allowedCollectionIds && !allowedCollectionIds.includes(collectionId)) { + return new NotFoundError() + } + const itemQuery = { collections: [collectionId], id: itemId } const { results } = await backend.search(itemQuery) const [item] = results if (!item) { - return new Error('Item not found') + return new NotFoundError() } const thumbnailAsset = Object.values(item.assets || []).find( @@ -1271,7 +1311,7 @@ const getItemThumbnail = async function (collectionId, itemId, backend) { ) if (!thumbnailAsset) { - return new Error('Thumbnail not found') + return new NotFoundError() } let location @@ -1296,7 +1336,7 @@ const getItemThumbnail = async function (collectionId, itemId, backend) { expiresIn: 60 * 5, // expiry in seconds }) } else { - return new Error('Thumbnail not found') + return new NotFoundError() } return { location } diff --git a/src/lib/database.js b/src/lib/database.js index 47a2bcbf..c0ebf868 100644 --- a/src/lib/database.js +++ b/src/lib/database.js @@ -1,7 +1,7 @@ import { isEmpty } from 'lodash-es' import { dbClient as _client, createIndex } from './database-client.js' import logger from './logger.js' -import { ValidationError } from './errors.js' +import { NotFoundError, ValidationError } from './errors.js' import { bboxToPolygon } from './geo-utils.js' const COLLECTIONS_INDEX = process.env['COLLECTIONS_INDEX'] || 'collections' @@ -666,7 +666,7 @@ async function getCollection(collectionId) { if (Array.isArray(response.body.hits.hits) && response.body.hits.hits.length) { return response.body.hits.hits[0]._source } - return new Error('Collection not found') + return new NotFoundError() } // get all collections diff --git a/src/lib/errors.js b/src/lib/errors.js index 86e77d81..5a7f0d65 100644 --- a/src/lib/errors.js +++ b/src/lib/errors.js @@ -1,7 +1,15 @@ +/* eslint-disable max-classes-per-file */ /* eslint-disable import/prefer-default-export */ export class ValidationError extends Error { constructor(message) { super(message) - this.name = 'ValidationError' + this.name = this.constructor.name + } +} + +export class NotFoundError extends Error { + constructor(message) { + super(message) + this.name = this.constructor.name } } diff --git a/src/lib/s3-utils.ts b/src/lib/s3-utils.ts index fea6c344..cfea2575 100644 --- a/src/lib/s3-utils.ts +++ b/src/lib/s3-utils.ts @@ -14,7 +14,7 @@ const getObjectBody = async (s3Location: {bucket: string, key: string}) => { return result.Body } catch (error) { if (error instanceof Error) { - console.log(`Failed to fetch ${s3Location.bucket}/${s3Location.key}: ${error.message}`) + console.error(`Failed to fetch ${s3Location.bucket}/${s3Location.key}: ${error.message}`) } throw error } diff --git a/tests/system/test-api-collection-items-get.js b/tests/system/test-api-collection-items-get.js index 0969ae43..1c094ea3 100644 --- a/tests/system/test-api-collection-items-get.js +++ b/tests/system/test-api-collection-items-get.js @@ -91,3 +91,24 @@ test('GET /collections/:collectionId/items for non-existent collection returns 4 t.is(response.statusCode, 404) }) + +test('GET /collections/:collectionId/items with restriction returns filtered collections', async (t) => { + const { collectionId } = t.context + + const path = `collections/${collectionId}/items` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-get-aggregate.js b/tests/system/test-api-get-aggregate.js index 33179f71..1497d97a 100644 --- a/tests/system/test-api-get-aggregate.js +++ b/tests/system/test-api-get-aggregate.js @@ -500,3 +500,91 @@ test('GET /aggregate with aggregations and filter params', async (t) => { value: 1 }]) }) + +test('GET /aggregate with restriction returns filtered collections', async (t) => { + const fixtureFiles = [ + 'collection.json', + 'LC80100102015050LGN00.json', + 'LC80100102015082LGN00.json' + ] + const items = await Promise.all(fixtureFiles.map((x) => loadJson(x))) + await ingestItems(items) + await refreshIndices() + + const collectionId = 'landsat-8-l1' + + let response = null + + // validate how many items we have total without restricting collections + response = await t.context.api.client.get( + 'aggregate', + { + searchParams: new URLSearchParams({ + aggregations: ['total_count'] + }), + resolveBodyOnly: false, + } + ) + + t.is(response.statusCode, 200) + t.deepEqual(response.body.aggregations, [{ + name: 'total_count', + data_type: 'integer', + value: 4 + }]) + + // get the counts for collectionId without restrictions + response = await t.context.api.client.get( + 'aggregate', + { + searchParams: new URLSearchParams({ + aggregations: ['total_count'], + collections: [collectionId] + }), + resolveBodyOnly: false, + } + ) + + t.is(response.statusCode, 200) + t.deepEqual(response.body.aggregations, [{ + name: 'total_count', + data_type: 'integer', + value: 2 + }]) + + // restrict collections to include the one we just got 2 results for + + const response2 = await t.context.api.client.get( + 'aggregate', + { + searchParams: new URLSearchParams({ + aggregations: ['total_count'], + _collections: [collectionId, 'foo', 'bar'] + }), + resolveBodyOnly: false, + } + ) + + t.is(response.statusCode, 200) + t.deepEqual(response2.body.aggregations, response.body.aggregations) + + // restrict collections to a non-existent one with no items, so should be 0 results + response = await t.context.api.client.get( + 'aggregate', + { + searchParams: new URLSearchParams({ + aggregations: ['total_count'], + collections: [collectionId], + _collections: ['not-a-collection'] + }), + resolveBodyOnly: false, + } + ) + + t.is(response.statusCode, 200) + t.deepEqual(response.body.aggregations, [{ + name: 'total_count', + data_type: 'integer', + value: 0 + }]) +}) diff --git a/tests/system/test-api-get-collection-aggregate.js b/tests/system/test-api-get-collection-aggregate.js index aea438fd..c3550db2 100644 --- a/tests/system/test-api-get-collection-aggregate.js +++ b/tests/system/test-api-get-collection-aggregate.js @@ -167,3 +167,24 @@ test('GET /aggregate with aggregation not supported by this collection', async ( description: 'Aggregation grid_code_frequency not supported by collection sentinel-1-grd' }) }) + +test('GET /collections/:collectionId/aggregate with restriction returns filtered collections', async (t) => { + const collectionId = 'landsat-8-l1' + + const path = `collections/${collectionId}/aggregate` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-get-collection-aggregations.js b/tests/system/test-api-get-collection-aggregations.js index c4dd1c23..19cba775 100644 --- a/tests/system/test-api-get-collection-aggregations.js +++ b/tests/system/test-api-get-collection-aggregations.js @@ -129,3 +129,24 @@ test('GET /collections/:collectionId/aggregations returns default aggregations f ]) t.deepEqual(response.body.links, links(proto, host, collectionId)) }) + +test('GET /collections/:collectionId/aggregations with restriction returns filtered collections', async (t) => { + const { collectionId } = t.context + + const path = `collections/${collectionId}/aggregations` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-get-collection-queryables.js b/tests/system/test-api-get-collection-queryables.js index 74df64da..03f08b8a 100644 --- a/tests/system/test-api-get-collection-queryables.js +++ b/tests/system/test-api-get-collection-queryables.js @@ -111,3 +111,24 @@ test.only('GET /collection/:collectionId/queryables for collection with unsuppor t.regex(error.response.body.description, /.*Unsupported additionalProperties value: "false". Must be set to "true".*/) }) + +test('GET /collections/:collectionId/queryables with restriction returns filtered collections', async (t) => { + const { collectionId } = t.context + + const path = `collections/${collectionId}/queryables` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-get-collection.js b/tests/system/test-api-get-collection.js index 93f8444c..2af14698 100644 --- a/tests/system/test-api-get-collection.js +++ b/tests/system/test-api-get-collection.js @@ -59,3 +59,22 @@ test('GET /collection/:collectionId for non-existent collection returns Not Foun t.is(response.statusCode, 404) }) + +test('GET /collections/:collectionId with restriction returns filtered collections', async (t) => { + const { collectionId } = t.context + + t.is((await t.context.api.client.get(`collections/${collectionId}`, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(`collections/${collectionId}`, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(`collections/${collectionId}`, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-get-collections.js b/tests/system/test-api-get-collections.js index 97189b4e..bed1e7cf 100644 --- a/tests/system/test-api-get-collections.js +++ b/tests/system/test-api-get-collections.js @@ -12,11 +12,11 @@ test.before(async (t) => { t.context = standUpResult - const collectionId = randomId('collection') + t.context.collectionId = randomId('collection') const collection = await loadFixture( 'landsat-8-l1-collection.json', - { id: collectionId } + { id: t.context.collectionId } ) await ingestItem({ @@ -49,3 +49,21 @@ test('GET /collections has a content type of "application/json', async (t) => { t.is(response.headers['content-type'], 'application/json; charset=utf-8') }) + +test('GET /collections with restriction returns filtered collections', async (t) => { + await refreshIndices() + + const { collectionId } = t.context + + t.is((await t.context.api.client.get( + 'collections', { + searchParams: { _collections: `${collectionId},foo,bar` } } + ) + ).collections.length, 1) + + t.is((await t.context.api.client.get( + 'collections', { + searchParams: { _collections: 'not-a-collection' } } + ) + ).collections.length, 0) +}) diff --git a/tests/system/test-api-item-get.js b/tests/system/test-api-item-get.js index 64c442d5..81322f13 100644 --- a/tests/system/test-api-item-get.js +++ b/tests/system/test-api-item-get.js @@ -61,7 +61,7 @@ test('GET /collections/:collectionId/items/:itemId', async (t) => { t.is(response.body.collection, collectionId) }) -test('GET /collections/:collectionId/items/:itemId for a non-existent id returns Not Found"', async (t) => { +test('GET /collections/:collectionId/items/:itemId for a non-existent id returns not found', async (t) => { const { collectionId } = t.context const response = await t.context.api.client.get( @@ -71,3 +71,57 @@ test('GET /collections/:collectionId/items/:itemId for a non-existent id returns t.is(response.statusCode, 404) }) + +test('GET /collections/:collectionId/items/:itemId for a non-existent collection returns not found', async (t) => { + const response = await t.context.api.client.get( + 'collections/DOES_NOT_EXIST/items/DOES_NOT_EXIST', + { resolveBodyOnly: false, throwHttpErrors: false } + ) + + t.is(response.statusCode, 404) +}) + +test('GET /collections/:collectionId/items/:itemId with restriction returns filtered collections', async (t) => { + const { collectionId, itemId } = t.context + + const path = `collections/${collectionId}/items/${itemId}` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 200) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) + +test('GET /collections/:collectionId/items/:itemId/thumbnail with restriction returns filtered collections', async (t) => { + const { collectionId, itemId } = t.context + + const path = `collections/${collectionId}/items/${itemId}/thumbnail` + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, followRedirect: false + })).statusCode, 302) + + t.is((await t.context.api.client.get(path, + { + resolveBodyOnly: false, + followRedirect: false, + searchParams: { _collections: `${collectionId},foo,bar` } + })).statusCode, 302) + + t.is((await t.context.api.client.get(path, + { resolveBodyOnly: false, + followRedirect: false, + throwHttpErrors: false, + searchParams: { _collections: 'not-a-collection' } + })).statusCode, 404) +}) diff --git a/tests/system/test-api-search-get.js b/tests/system/test-api-search-get.js index ab661318..af607dac 100644 --- a/tests/system/test-api-search-get.js +++ b/tests/system/test-api-search-get.js @@ -142,3 +142,40 @@ test('/search filter, query, and item search in single request', async (t) => { }) t.is(response.features.length, 1) }) + +test('GET /search with restriction returns filtered collections', async (t) => { + const fixtureFiles = [ + 'catalog.json', + 'collection.json', + 'LC80100102015050LGN00.json', + 'LC80100102015082LGN00.json' + ] + const items = await Promise.all(fixtureFiles.map((x) => loadJson(x))) + await ingestItems(items) + await refreshIndices() + + const collectionId = 'landsat-8-l1' + const path = 'search' + + const r1 = await t.context.api.client.get(path, + { resolveBodyOnly: false }) + + t.is(r1.statusCode, 200) + t.is(r1.body.features.length, 3) + + const r2 = await t.context.api.client.get(path, + { resolveBodyOnly: false, + searchParams: { _collections: `${collectionId},foo,bar` } + }) + + t.is(r2.statusCode, 200) + t.is(r2.body.features.length, 2) + + const r3 = await t.context.api.client.get(path, + { resolveBodyOnly: false, + searchParams: { _collections: 'not-a-collection' } + }) + + t.is(r3.statusCode, 200) + t.is(r3.body.features.length, 0) +}) diff --git a/tests/system/test-api-search-post.js b/tests/system/test-api-search-post.js index c546e40c..a95bab96 100644 --- a/tests/system/test-api-search-post.js +++ b/tests/system/test-api-search-post.js @@ -1359,3 +1359,45 @@ test('/search - filter extension - s_intersects - non-existent geometry type', a // eslint-disable-next-line max-len /.*Operand for 's_intersects' must be a GeoJSON geometry: type was 'notPolygon'*/) }) + +test('POST /search with restriction returns filtered collections', async (t) => { + const fixtureFiles = [ + 'collection.json', + 'LC80100102015050LGN00.json', + 'LC80100102015082LGN00.json' + ] + const items = await Promise.all(fixtureFiles.map((x) => loadJson(x))) + await ingestItems(items) + await refreshIndices() + + const collectionId = 'landsat-8-l1' + const urlpath = 'search' + + const r1 = await t.context.api.client.post(urlpath, + { resolveBodyOnly: false, + json: { + _collections: [collectionId] + } }) + + t.is(r1.statusCode, 200) + t.is(r1.body.features.length, 2) + + const r2 = await t.context.api.client.post(urlpath, + { resolveBodyOnly: false, + json: { + _collections: [collectionId, 'foo', 'bar'] + } }) + + t.is(r2.statusCode, 200) + t.is(r2.body.features.length, 2) + + const r3 = await t.context.api.client.post(urlpath, + { resolveBodyOnly: false, + json: { + _collections: ['not-a-collection'] + } + }) + + t.is(r3.statusCode, 200) + t.is(r3.body.features.length, 0) +}) From 8200249e64e4a613435a3f7d6dee2bbdbaab79ee Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Mon, 7 Apr 2025 15:43:51 -0400 Subject: [PATCH 2/2] add ENABLE_COLLECTIONS_AUTHX --- CHANGELOG.md | 2 +- README.md | 5 ++++- src/lib/api.js | 4 +++- tests/system/test-api-collection-items-get.js | 1 + tests/system/test-api-get-aggregate.js | 2 ++ tests/system/test-api-get-collection-aggregate.js | 2 ++ .../test-api-get-collection-aggregations.js | 2 ++ .../system/test-api-get-collection-queryables.js | 2 ++ tests/system/test-api-get-collection.js | 2 ++ tests/system/test-api-get-collections.js | 15 +++++++++++++++ tests/system/test-api-item-get.js | 4 ++++ tests/system/test-api-search-get.js | 2 ++ tests/system/test-api-search-post.js | 2 ++ 13 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b9c3bd5..ec4dbff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - To all endpoints that depend on collections, add support for a query parameter (GET) or body field (POST) `_collections` that will filter to only those collections, but - will not reveal that in link contents. + will not reveal that in link contents. This is controlled by the "ENABLE_COLLECTIONS_AUTHX" ## [3.11.0] - 2025-03-27 diff --git a/README.md b/README.md index 5d33052e..503f2c3d 100644 --- a/README.md +++ b/README.md @@ -583,7 +583,7 @@ There are some settings that should be reviewed and updated as needeed in the se | REQUEST_LOGGING_FORMAT | Express request logging format to use. Any of the [Morgan predefined formats](https://github.com/expressjs/morgan#predefined-formats). | tiny | | STAC_API_URL | The root endpoint of this API | Inferred from request | | ENABLE_TRANSACTIONS_EXTENSION | Boolean specifying if the [Transaction Extension](https://github.com/radiantearth/stac-api-spec/tree/master/ogcapi-features/extensions/transaction) should be activated | false | -| ENABLE_CONTEXT_EXTENSION | Boolean specifying if the [Context Extension](https://github.com/stac-api-extensions/context) should be activated | false | +| ENABLE_CONTEXT_EXTENSION | Boolean specifying if the [Context Extension](https://github.com/stac-api-extensions/context) should be activated | false | | STAC_API_ROOTPATH | The path to append to URLs if this is not deployed at the server root. For example, if the server is deployed without a custom domain name, it will have the stage name (e.g., dev) in the path. | "" | | PRE_HOOK | The name of a Lambda function to be called as the pre-hook. | none | | POST_HOOK | The name of a Lambda function to be called as the post-hook. | none | @@ -598,6 +598,7 @@ There are some settings that should be reviewed and updated as needeed in the se | CORS_CREDENTIALS | Configure whether or not to send the `Access-Control-Allow-Credentials` CORS header. Header will be sent if set to `true`. | none | | CORS_METHODS | Configure whether or not to send the `Access-Control-Allow-Methods` CORS header. Expects a comma-delimited string, e.g., `GET,PUT,POST`. | `GET,HEAD,PUT,PATCH,POST,DELETE` | | CORS_HEADERS | Configure whether or not to send the `Access-Control-Allow-Headers` CORS header. Expects a comma-delimited string, e.g., `Content-Type,Authorization`. If not specified, defaults to reflecting the headers specified in the request’s `Access-Control-Request-Headers` header. | none | +| ENABLE_COLLECTIONS_AUTHX | Enables support for hidden `_collections` query parameter / field when set to `true`. | none | Additionally, the credential for OpenSearch must be configured, as decribed in the section [Populating and accessing credentials](#populating-and-accessing-credentials). @@ -1110,6 +1111,8 @@ parameter named (for GET requests) or body JSON field (for POST requests) named to filter the collections a user has access to. This parameter/field will be excluded from pagination links, so it does not need to be removed on egress. +This feature must be enabled with the `ENABLE_COLLECTIONS_AUTHX` configuration. + The endpoints this applies to are: - /collections diff --git a/src/lib/api.js b/src/lib/api.js index 9ae70712..3e473aa8 100644 --- a/src/lib/api.js +++ b/src/lib/api.js @@ -336,7 +336,9 @@ const extractIds = function (params) { } const extractAllowedCollectionIds = function (params) { - return parseIds(params._collections) + return process.env['ENABLE_COLLECTIONS_AUTHX'] === 'true' + ? parseIds(params._collections) + : undefined } const extractCollectionIds = function (params) { diff --git a/tests/system/test-api-collection-items-get.js b/tests/system/test-api-collection-items-get.js index 1c094ea3..d7f88690 100644 --- a/tests/system/test-api-collection-items-get.js +++ b/tests/system/test-api-collection-items-get.js @@ -94,6 +94,7 @@ test('GET /collections/:collectionId/items for non-existent collection returns 4 test('GET /collections/:collectionId/items with restriction returns filtered collections', async (t) => { const { collectionId } = t.context + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' const path = `collections/${collectionId}/items` diff --git a/tests/system/test-api-get-aggregate.js b/tests/system/test-api-get-aggregate.js index 1497d97a..48a97842 100644 --- a/tests/system/test-api-get-aggregate.js +++ b/tests/system/test-api-get-aggregate.js @@ -502,6 +502,8 @@ test('GET /aggregate with aggregations and filter params', async (t) => { }) test('GET /aggregate with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const fixtureFiles = [ 'collection.json', 'LC80100102015050LGN00.json', diff --git a/tests/system/test-api-get-collection-aggregate.js b/tests/system/test-api-get-collection-aggregate.js index c3550db2..d3f8e58c 100644 --- a/tests/system/test-api-get-collection-aggregate.js +++ b/tests/system/test-api-get-collection-aggregate.js @@ -169,6 +169,8 @@ test('GET /aggregate with aggregation not supported by this collection', async ( }) test('GET /collections/:collectionId/aggregate with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const collectionId = 'landsat-8-l1' const path = `collections/${collectionId}/aggregate` diff --git a/tests/system/test-api-get-collection-aggregations.js b/tests/system/test-api-get-collection-aggregations.js index 19cba775..75d8781f 100644 --- a/tests/system/test-api-get-collection-aggregations.js +++ b/tests/system/test-api-get-collection-aggregations.js @@ -131,6 +131,8 @@ test('GET /collections/:collectionId/aggregations returns default aggregations f }) test('GET /collections/:collectionId/aggregations with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const { collectionId } = t.context const path = `collections/${collectionId}/aggregations` diff --git a/tests/system/test-api-get-collection-queryables.js b/tests/system/test-api-get-collection-queryables.js index 03f08b8a..bf818931 100644 --- a/tests/system/test-api-get-collection-queryables.js +++ b/tests/system/test-api-get-collection-queryables.js @@ -113,6 +113,8 @@ test.only('GET /collection/:collectionId/queryables for collection with unsuppor }) test('GET /collections/:collectionId/queryables with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const { collectionId } = t.context const path = `collections/${collectionId}/queryables` diff --git a/tests/system/test-api-get-collection.js b/tests/system/test-api-get-collection.js index 2af14698..8fc5bb33 100644 --- a/tests/system/test-api-get-collection.js +++ b/tests/system/test-api-get-collection.js @@ -61,6 +61,8 @@ test('GET /collection/:collectionId for non-existent collection returns Not Foun }) test('GET /collections/:collectionId with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const { collectionId } = t.context t.is((await t.context.api.client.get(`collections/${collectionId}`, diff --git a/tests/system/test-api-get-collections.js b/tests/system/test-api-get-collections.js index 852c8e2e..166ae0ba 100644 --- a/tests/system/test-api-get-collections.js +++ b/tests/system/test-api-get-collections.js @@ -51,10 +51,25 @@ test('GET /collections has a content type of "application/json', async (t) => { }) test('GET /collections with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + await refreshIndices() const { collectionId } = t.context + // disable collections filtering + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'not true' + + t.is((await t.context.api.client.get( + 'collections', { + searchParams: { _collections: 'not-a-collection' } } + ) + ).collections.length, 1) + + // enable collections filtering + + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + t.is((await t.context.api.client.get( 'collections', { searchParams: { _collections: `${collectionId},foo,bar` } } diff --git a/tests/system/test-api-item-get.js b/tests/system/test-api-item-get.js index 81322f13..c95ed332 100644 --- a/tests/system/test-api-item-get.js +++ b/tests/system/test-api-item-get.js @@ -82,6 +82,8 @@ test('GET /collections/:collectionId/items/:itemId for a non-existent collection }) test('GET /collections/:collectionId/items/:itemId with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const { collectionId, itemId } = t.context const path = `collections/${collectionId}/items/${itemId}` @@ -103,6 +105,8 @@ test('GET /collections/:collectionId/items/:itemId with restriction returns filt }) test('GET /collections/:collectionId/items/:itemId/thumbnail with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const { collectionId, itemId } = t.context const path = `collections/${collectionId}/items/${itemId}/thumbnail` diff --git a/tests/system/test-api-search-get.js b/tests/system/test-api-search-get.js index af607dac..e970bf2c 100644 --- a/tests/system/test-api-search-get.js +++ b/tests/system/test-api-search-get.js @@ -144,6 +144,8 @@ test('/search filter, query, and item search in single request', async (t) => { }) test('GET /search with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const fixtureFiles = [ 'catalog.json', 'collection.json', diff --git a/tests/system/test-api-search-post.js b/tests/system/test-api-search-post.js index 6d88eeb1..269d018d 100644 --- a/tests/system/test-api-search-post.js +++ b/tests/system/test-api-search-post.js @@ -1361,6 +1361,8 @@ test('/search - filter extension - s_intersects - non-existent geometry type', a }) test('POST /search with restriction returns filtered collections', async (t) => { + process.env['ENABLE_COLLECTIONS_AUTHX'] = 'true' + const fixtureFiles = [ 'collection.json', 'LC80100102015050LGN00.json',