diff --git a/lib/adapter/createPaging-test.js b/lib/adapter/createPaging-test.js new file mode 100644 index 0000000..f9607b8 --- /dev/null +++ b/lib/adapter/createPaging-test.js @@ -0,0 +1,143 @@ +import test from 'ava' + +import createPaging from './createPaging' + +// Helpers + +const prepareData = (data) => data.map((item) => ({...item, _id: `${item.type}:${item.id}`})) + +// Tests + +test('should return next: null when no data', (t) => { + const data = [] + const params = {type: 'entry', pageSize: 2} + const expected = {next: null} + + const ret = createPaging(data, params) + + t.deepEqual(ret, expected) +}) + +test('should return paging for first page', (t) => { + const data = prepareData([{id: 'ent1', type: 'entry'}, {id: 'ent2', type: 'entry'}]) + const params = { + type: 'entry', + pageSize: 2 + } + const expected = { + next: { + type: 'entry', + query: {_id: {$gte: 'entry:ent2'}}, + pageAfter: 'entry:ent2', + pageSize: 2 + } + } + + const ret = createPaging(data, params) + + t.deepEqual(ret, expected) +}) + +test('should return paging for second page', (t) => { + const data = prepareData([{id: 'ent3', type: 'entry'}, {id: 'ent4', type: 'entry'}]) + const params = { + type: 'entry', + query: {_id: {$gte: 'entry:ent2'}}, + pageAfter: 'entry:ent2', + pageSize: 2 + } + const expected = { + next: { + type: 'entry', + query: {_id: {$gte: 'entry:ent4'}}, + pageAfter: 'entry:ent4', + pageSize: 2 + } + } + + const ret = createPaging(data, params) + + t.deepEqual(ret, expected) +}) + +test('should return paging when sorting', (t) => { + const data = prepareData([ + {id: 'ent2', type: 'entry', attributes: {index: 1}}, + {id: 'ent3', type: 'entry', attributes: {index: 2}} + ]) + const params = { + type: 'entry', + pageSize: 2 + } + const sort = { + 'attributes.index': 1 + } + const expected = { + next: { + type: 'entry', + query: {'attributes.index': {$gte: 2}}, + pageAfter: 'entry:ent3', + pageSize: 2 + } + } + + const ret = createPaging(data, params, sort) + + t.deepEqual(ret, expected) +}) + +test('should return paging when sorting descending', (t) => { + const data = prepareData([ + {id: 'ent3', type: 'entry', attributes: {index: 2}}, + {id: 'ent2', type: 'entry', attributes: {index: 1}} + ]) + const params = { + type: 'entry', + pageSize: 2 + } + const sort = { + 'attributes.index': -1 + } + const expected = { + next: { + type: 'entry', + query: {'attributes.index': {$lte: 1}}, + pageAfter: 'entry:ent2', + pageSize: 2 + } + } + + const ret = createPaging(data, params, sort) + + t.deepEqual(ret, expected) +}) + +test('should return paging when sorting ascending and descending', (t) => { + const data = prepareData([ + {id: 'ent3', type: 'entry', attributes: {index: 2}}, + {id: 'ent2', type: 'entry', attributes: {index: 1}} + ]) + const params = { + type: 'entry', + pageSize: 2 + } + const sort = { + 'attributes.index': -1, + id: 1 + } + const expected = { + next: { + type: 'entry', + query: { + 'attributes.index': {$lte: 1}, + id: {$gte: 'ent2'} + }, + pageAfter: 'entry:ent2', + pageSize: 2 + } + } + + const ret = createPaging(data, params, sort) + + t.deepEqual(ret, expected) +}) diff --git a/lib/adapter/createPaging.js b/lib/adapter/createPaging.js new file mode 100644 index 0000000..9436ff2 --- /dev/null +++ b/lib/adapter/createPaging.js @@ -0,0 +1,35 @@ +const dotprop = require('dot-prop') + +const createQuery = (lastItem, sort) => { + if (sort) { + return Object.keys(sort).reduce((query, key) => { + const value = dotprop.get(lastItem, key) + const operator = (sort[key] > 0) ? '$gte' : '$lte' + return {...query, [key]: {[operator]: value}} + }, {}) + } else { + return {_id: {$gte: lastItem._id}} + } +} + +const createPaging = (data, params, sort) => { + if (data.length === 0) { + return {next: null} + } + const lastItem = data[data.length - 1] + + const {typePlural, ...nextParams} = params + + const query = createQuery(lastItem, sort) + + return { + next: { + ...nextParams, + pageSize: params.pageSize, + pageAfter: lastItem._id, + query + } + } +} + +module.exports = createPaging diff --git a/lib/adapter/getDocs.js b/lib/adapter/getDocs.js index f91036f..298bee7 100644 --- a/lib/adapter/getDocs.js +++ b/lib/adapter/getDocs.js @@ -1,35 +1,26 @@ const prepareFilter = require('./prepareFilter') +const createPaging = require('./createPaging') -const createPaging = (data, params) => { - if (data.length === 0) { - return {} +// Move the cursor to the first doc after the `pageAfter` +// When no `pageAfter`, just start from the beginning +const moveToData = async (cursor, pageAfter) => { + if (!pageAfter) { + // Start from the beginning + return true } - const lastId = data[data.length - 1]._id - const {typePlural, ...nextParams} = params + let doc + do { + doc = await cursor.next() + } while (doc && doc._id !== pageAfter) - return { - next: { - ...nextParams, - pageSize: params.pageSize, - pageAfter: lastId, - query: {_id: {$gte: lastId}} - } - } + return !!doc // false if the doc to start after is not found } -const getPage = async (cursor, {pageSize = Infinity, pageAfter}) => { +// Get one page of docs from where the cursor is +const getData = async (cursor, pageSize) => { const data = [] - // When pageAfter is set – loop until we find the doc with that _id - if (pageAfter) { - let doc - do { - doc = await cursor.next() - } while (doc && doc._id !== pageAfter) - } - - // Get the number of docs specified with pageSize - or the rest of the docs while (data.length < pageSize) { const doc = await cursor.next() if (!doc) { @@ -41,6 +32,18 @@ const getPage = async (cursor, {pageSize = Infinity, pageAfter}) => { return data } +const getPage = async (cursor, {pageSize = Infinity, pageAfter}) => { + // When pageAfter is set – loop until we find the doc with that _id + const foundFirst = moveToData(cursor, pageAfter) + + // Get the number of docs specified with pageSize - or the rest of the docs + if (foundFirst) { + return getData(cursor, pageSize) + } + + return [] +} + async function getDocs (getCollection, {endpoint, params}) { const collection = getCollection() @@ -61,7 +64,7 @@ async function getDocs (getCollection, {endpoint, params}) { const response = {status: 'ok', data} if (params.pageSize) { - response.paging = createPaging(data, params) + response.paging = createPaging(data, params, endpoint.sort) } return response diff --git a/lib/adapter/prepareFilter.js b/lib/adapter/prepareFilter.js index 225bcb5..00d1b11 100644 --- a/lib/adapter/prepareFilter.js +++ b/lib/adapter/prepareFilter.js @@ -1,5 +1,15 @@ const dotprop = require('dot-prop') +const setTypeOrId = (query, type, id) => { + if (Object.keys(query).length === 0) { + if (id) { + query._id = `${type}:${id}` + } else { + query.type = type + } + } +} + const prepareFilter = ({type, id}, {query: queryProps = []}, params) => { // Create query object from array of props const query = queryProps.reduce((filter, prop) => { @@ -8,13 +18,7 @@ const prepareFilter = ({type, id}, {query: queryProps = []}, params) => { }, {}) // Set query props from id and type if no query was provided - if (Object.keys(query).length === 0) { - if (id) { - query._id = `${type}:${id}` - } else { - query.type = type - } - } + setTypeOrId(query, type, id) // Add query from payload params if (params && params.query) { diff --git a/tests/get-test.js b/tests/get-test.js index 624eaa1..4c4d6f0 100644 --- a/tests/get-test.js +++ b/tests/get-test.js @@ -116,89 +116,6 @@ test('should get a document with endpoint query', async (t) => { t.is(data[0].id, 'ent2') }) -test('should get one page of documents with params for next page', async (t) => { - const {collection, collectionName} = t.context - await insertDocuments(collection, [ - {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, - {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, - {_id: 'entry:ent3', id: 'ent2', type: 'entry'} - ]) - const request = { - action: 'GET', - params: { - type: 'entry', - pageSize: 2 - }, - endpoint: { - collection: collectionName, - db: 'test' - } - } - const expectedPaging = { - next: { - type: 'entry', - query: {_id: {$gte: 'entry:ent2'}}, - pageAfter: 'entry:ent2', - pageSize: 2 - } - } - - const connection = await adapter.connect({sourceOptions}) - const response = await adapter.send(request, connection) - await adapter.disconnect(connection) - - t.truthy(response) - t.is(response.status, 'ok') - const {data} = response - t.is(data.length, 2) - t.is(data[0].id, 'ent1') - t.is(data[1].id, 'ent2') - t.deepEqual(response.paging, expectedPaging) -}) - -test('should get second page of documents', async (t) => { - const {collection, collectionName} = t.context - await insertDocuments(collection, [ - {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, - {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, - {_id: 'entry:ent3', id: 'ent3', type: 'entry'}, - {_id: 'entry:ent4', id: 'ent4', type: 'entry'} - ]) - const request = { - action: 'GET', - params: { - type: 'entry', - query: {_id: {$gte: 'entry:ent2'}}, - pageAfter: 'entry:ent2', - pageSize: 2 - }, - endpoint: { - collection: collectionName, - db: 'test' - } - } - const expectedPaging = { - next: { - type: 'entry', - query: {_id: {$gte: 'entry:ent4'}}, - pageAfter: 'entry:ent4', - pageSize: 2 - } - } - - const connection = await adapter.connect({sourceOptions}) - const response = await adapter.send(request, connection) - await adapter.disconnect(connection) - - t.truthy(response) - t.is(response.status, 'ok') - const {data} = response - t.is(data.length, 2) - t.is(data[0].id, 'ent3') - t.is(data[1].id, 'ent4') - t.deepEqual(response.paging, expectedPaging) -}) - test('should sort documents', async (t) => { const {collection, collectionName} = t.context await insertDocuments(collection, [ diff --git a/tests/paging-test.js b/tests/paging-test.js new file mode 100644 index 0000000..0865597 --- /dev/null +++ b/tests/paging-test.js @@ -0,0 +1,353 @@ +import test from 'ava' +import { + baseUri, + openMongoWithCollection, + closeMongo, + insertDocuments, + deleteDocuments +} from './helpers/mongo' + +import mongodb from '..' +const {adapter} = mongodb + +// Helpers + +const sourceOptions = {baseUri} + +test.beforeEach(async (t) => { + t.context = await openMongoWithCollection('test') +}) + +test.afterEach.always(async (t) => { + const {client, collection} = t.context + deleteDocuments(collection, {type: 'entry'}) + closeMongo(client) +}) + +// Tests + +test('should get one page of documents with params for next page', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, + {_id: 'entry:ent3', id: 'ent2', type: 'entry'} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test' + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {_id: {$gte: 'entry:ent2'}}, + pageAfter: 'entry:ent2', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 2) + t.is(data[0].id, 'ent1') + t.is(data[1].id, 'ent2') + t.deepEqual(response.paging, expectedPaging) +}) + +test('should get second page of documents', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry'}, + {_id: 'entry:ent4', id: 'ent4', type: 'entry'} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {_id: {$gte: 'entry:ent2'}}, + pageAfter: 'entry:ent2', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test' + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {_id: {$gte: 'entry:ent4'}}, + pageAfter: 'entry:ent4', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 2) + t.is(data[0].id, 'ent3') + t.is(data[1].id, 'ent4') + t.deepEqual(response.paging, expectedPaging) +}) + +test('should return less than a full page at the end', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry'} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {_id: {$gte: 'entry:ent2'}}, + pageAfter: 'entry:ent2', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test' + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {_id: {$gte: 'entry:ent3'}}, + pageAfter: 'entry:ent3', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 1) + t.is(data[0].id, 'ent3') + t.deepEqual(response.paging, expectedPaging) +}) + +test('should return empty array when past last page', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry'}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry'}, + {_id: 'entry:ent4', id: 'ent4', type: 'entry'} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {_id: {$gte: 'entry:ent4'}}, + pageAfter: 'entry:ent4', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test' + } + } + const expectedPaging = { + next: null + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 0) + t.deepEqual(response.paging, expectedPaging) +}) + +test('should not throw when pageAfter does not exist', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry'}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry'} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {_id: {$gte: 'entry:ent3'}}, + pageAfter: 'entry:ent3', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test' + } + } + const expectedPaging = { + next: null + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 0) + t.deepEqual(response.paging, expectedPaging) +}) + +test('should get second page of documents when sorting', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry', attributes: {index: 3}}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry', attributes: {index: 1}}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry', attributes: {index: 2}}, + {_id: 'entry:ent4', id: 'ent4', type: 'entry', attributes: {index: 4}} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {'attributes.index': {$gte: 2}}, + pageAfter: 'entry:ent3', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test', + sort: {'attributes.index': 1} + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {'attributes.index': {$gte: 4}}, + pageAfter: 'entry:ent4', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 2) + t.is(data[0].id, 'ent1') + t.is(data[1].id, 'ent4') + t.deepEqual(response.paging, expectedPaging) +}) + +test('should get second page of documents when sorting descending', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry', attributes: {index: 3}}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry', attributes: {index: 1}}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry', attributes: {index: 2}}, + {_id: 'entry:ent4', id: 'ent4', type: 'entry', attributes: {index: 4}} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {'attributes.index': {$lte: 3}}, + pageAfter: 'entry:ent1', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test', + sort: {'attributes.index': -1} + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {'attributes.index': {$lte: 1}}, + pageAfter: 'entry:ent2', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 2) + t.is(data[0].id, 'ent3') + t.is(data[1].id, 'ent2') + t.deepEqual(response.paging, expectedPaging) +}) + +test('should get second page of documents when sorting key is not unique', async (t) => { + const {collection, collectionName} = t.context + await insertDocuments(collection, [ + {_id: 'entry:ent1', id: 'ent1', type: 'entry', attributes: {index: 2}}, + {_id: 'entry:ent2', id: 'ent2', type: 'entry', attributes: {index: 1}}, + {_id: 'entry:ent3', id: 'ent3', type: 'entry', attributes: {index: 1}}, + {_id: 'entry:ent4', id: 'ent4', type: 'entry', attributes: {index: 3}} + ]) + const request = { + action: 'GET', + params: { + type: 'entry', + query: {'attributes.index': {$gte: 1}}, + pageAfter: 'entry:ent3', + pageSize: 2 + }, + endpoint: { + collection: collectionName, + db: 'test', + sort: {'attributes.index': 1} + } + } + const expectedPaging = { + next: { + type: 'entry', + query: {'attributes.index': {$gte: 3}}, + pageAfter: 'entry:ent4', + pageSize: 2 + } + } + + const connection = await adapter.connect({sourceOptions}) + const response = await adapter.send(request, connection) + await adapter.disconnect(connection) + + t.truthy(response) + t.is(response.status, 'ok') + const {data} = response + t.is(data.length, 2) + t.is(data[0].id, 'ent1') + t.is(data[1].id, 'ent4') + t.deepEqual(response.paging, expectedPaging) +})