fix: upload list pagination headers (#1739)

* fix: upload list pagination headers * fix: test * fix: separate components * fix: use specific type of pagination * fix: all the things * fix: db tests * fix: api tests * fix: cron tests * fix: website tests * chore: appease website linter * chore: fix missed merge conflict. * chore: fix ilinting * chore: rm unessesary failing test * chore: fix linting on website. * chore: more linting fixes! Co-authored-by: joshJarr <josh.jarvis@potatolondon.com>
web3-storage · Aug 30, 2022 · 2ffe6d7 · 2ffe6d7
1 parent 285b76e
commit 2ffe6d7
Show file tree

Hide file tree

Showing 27 changed files with 1,358 additions and 588 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/api/src/errors.js b/packages/api/src/errors.js
@@ -205,3 +205,12 @@ export class PSAErrorDB extends PinningServiceApiError {
   }
 }
 PSAErrorDB.CODE = 'PSA_DB_ERROR'
+
+export class RangeNotSatisfiableError extends HTTPError {
+  constructor (msg = 'Range Not Satisfiable') {
+    super(msg, 416)
+    this.name = 'RangeNotSatisfiableError'
+    this.code = RangeNotSatisfiableError.CODE
+  }
+}
+RangeNotSatisfiableError.CODE = 'ERROR_RANGE_NOT_SATISFIABLE'
diff --git a/packages/api/src/user.js b/packages/api/src/user.js
@@ -1,7 +1,7 @@
 import * as JWT from './utils/jwt.js'
 import { JSONResponse } from './utils/json-response.js'
 import { JWT_ISSUER } from './constants.js'
-import { HTTPError } from './errors.js'
+import { HTTPError, RangeNotSatisfiableError } from './errors.js'
 import { getTagValue, hasPendingTagProposal, hasTag } from './utils/tags.js'
 import {
   NO_READ_OR_WRITE,
@@ -19,6 +19,7 @@ import { magicLinkBypassForE2ETestingInTestmode } from './magic.link.js'
  * @typedef {{ _id: string, name: string }} AuthToken
  * @typedef {{ user: User authToken?: AuthToken }} Auth
  * @typedef {Request & { auth: Auth }} AuthenticatedRequest
+ * @typedef {import('@web3-storage/db').PageRequest} PageRequest
  */
 
 /**
@@ -263,38 +264,86 @@ export async function userUploadsGet (request, env) {
   const requestUrl = new URL(request.url)
   const { searchParams } = requestUrl
 
-  const { size, page, offset, before, after, sortBy, sortOrder } = pagination(searchParams)
+  const pageRequest = pagination(searchParams)
 
-  const data = await env.db.listUploads(request.auth.user._id, {
-    size,
-    offset,
-    before,
-    after,
-    sortBy,
-    sortOrder
-  })
+  let data
+  try {
+    data = await env.db.listUploads(request.auth.user._id, pageRequest)
+  } catch (err) {
+    if (err.code === 'RANGE_NOT_SATISFIABLE_ERROR_DB') {
+      throw new RangeNotSatisfiableError()
+    }
+    throw err
+  }
 
-  let link = ''
-  // If there's more results to show...
-  if (page > 1) {
-    link += `<${requestUrl.pathname}?size=${size}&page=${page - 1}>; rel="previous"`
+  const headers = { Count: data.count }
+
+  if (pageRequest.size != null) {
+    headers.Size = pageRequest.size // Deprecated, use Link header instead.
   }
 
-  if (data.uploads.length + offset < data.count) {
-    if (link !== '') link += ', '
-    link += `<${requestUrl.pathname}?size=${size}&page=${page + 1}>; rel="next"`
+  if (pageRequest.page != null) {
+    headers.Page = pageRequest.page // Deprecated, use Link header instead.
   }
 
-  const headers = {
-    Count: data.count,
-    Size: size,
-    Page: page,
-    Link: link
+  const link = getLinkHeader({
+    url: requestUrl.pathname,
+    pageRequest,
+    items: data.uploads,
+    count: data.count
+  })
+
+  if (link) {
+    headers.Link = link
   }
 
   return new JSONResponse(data.uploads, { headers })
 }
 
+/**
+ * Generates a HTTP `Link` header for the given page request and data.
+ *
+ * @param {Object} args
+ * @param {string|URL} args.url Base URL
+ * @param {PageRequest} args.pageRequest Details for the current page of data
+ * @param {Array<{ created: string }>} args.items Page items
+ * @param {number} args.count Total items available
+ */
+function getLinkHeader ({ url, pageRequest, items, count }) {
+  const rels = []
+
+  if ('before' in pageRequest) {
+    const { size } = pageRequest
+    if (items.length === size) {
+      const oldest = items[items.length - 1]
+      const nextParams = new URLSearchParams({ size, before: oldest.created })
+      rels.push(`<${url}?${nextParams}>; rel="next"`)
+    }
+  } else if ('page' in pageRequest) {
+    const { size, page } = pageRequest
+    const pages = Math.ceil(count / size)
+    if (page < pages) {
+      const nextParams = new URLSearchParams({ size, page: page + 1 })
+      rels.push(`<${url}?${nextParams}>; rel="next"`)
+    }
+
+    const lastParams = new URLSearchParams({ size, page: pages })
+    rels.push(`<${url}?${lastParams}>; rel="last"`)
+
+    const firstParams = new URLSearchParams({ size, page: 1 })
+    rels.push(`<${url}?${firstParams}>; rel="first"`)
+
+    if (page > 1) {
+      const prevParams = new URLSearchParams({ size, page: page - 1 })
+      rels.push(`<${url}?${prevParams}>; rel="previous"`)
+    }
+  } else {
+    throw new Error('unknown page request type')
+  }
+
+  return rels.join(', ')
+}
+
 /**
  * Delete an user upload. This actually raises a tombstone rather than
  * deleting it entirely.
@@ -341,7 +390,7 @@ export async function userPinsGet (request, env) {
   const requestUrl = new URL(request.url)
   const { searchParams } = requestUrl
 
-  const { size, page, offset, before, after, sortBy, sortOrder } = pagination(searchParams)
+  const pageRequest = pagination(searchParams)
   const urlParams = new URLSearchParams(requestUrl.search)
   const params = Object.fromEntries(urlParams)
 
@@ -357,36 +406,39 @@ export async function userPinsGet (request, env) {
   try {
     pinRequests = await env.db.listPsaPinRequests(tokens, {
       ...psaParams.data,
-      limit: size,
-      offset,
-      before,
-      after,
-      sortBy,
-      sortOrder
+      limit: pageRequest.size,
+      offset: pageRequest.size * (pageRequest.page - 1)
     })
-  } catch (e) {
-    console.error(e)
-    throw new HTTPError('No pinning resources found for user', 404)
+  } catch (err) {
+    if (err.code === 'RANGE_NOT_SATISFIABLE_ERROR_DB') {
+      throw new RangeNotSatisfiableError()
+    }
+    throw err
   }
 
   const pins = pinRequests.results.map((pinRequest) => toPinStatusResponse(pinRequest))
 
-  let link = ''
-  // If there's more results to show...
-  if (page > 1) {
-    link += `<${requestUrl.pathname}?size=${size}&page=${page - 1}>; rel="previous"`
+  const headers = {
+    Count: pinRequests.count
   }
 
-  if (pins.length + offset < pinRequests.count) {
-    if (link !== '') link += ', '
-    link += `<${requestUrl.pathname}?size=${size}&page=${page + 1}>; rel="next"`
+  if (pageRequest.size != null) {
+    headers.Size = pageRequest.size // Deprecated, use Link header instead.
   }
 
-  const headers = {
-    Count: pinRequests.count || 0,
-    Size: size,
-    Page: page,
-    Link: link
+  if (pageRequest.page != null) {
+    headers.Page = pageRequest.page // Deprecated, use Link header instead.
+  }
+
+  const link = getLinkHeader({
+    url: requestUrl.pathname,
+    pageRequest,
+    items: pinRequests.results,
+    count: pinRequests.count
+  })
+
+  if (link) {
+    headers.Link = link
   }
 
   return new JSONResponse({

diff --git a/packages/api/src/utils/pagination.js b/packages/api/src/utils/pagination.js
@@ -1,3 +1,8 @@
+/** @type {import('@web3-storage/db').SortField[]} */
+const sortableValues = ['Name', 'Date']
+/** @type {import('@web3-storage/db').SortOrder[]} */
+const sortableOrders = ['Asc', 'Desc']
+
 /**
  * Get a parsed and validated list of pagination properties to pass to the DB query.
  * This standard is used across the website
@@ -11,13 +16,10 @@
  * ```
  *
  * @param  {URLSearchParams} searchParams
- *
+ * @returns {import('@web3-storage/db').PageRequest}
  */
 export function pagination (searchParams) {
-  const sortableValues = ['Name', 'Date']
-  const sortableOrders = ['Asc', 'Desc']
-
-  let size = 25
+  let size
   if (searchParams.has('size')) {
     const parsedSize = parseInt(searchParams.get('size'))
     if (isNaN(parsedSize) || parsedSize <= 0 || parsedSize > 1000) {
@@ -26,53 +28,38 @@ export function pagination (searchParams) {
     size = parsedSize
   }
 
-  let offset = 0
-  let page = 1
-  if (searchParams.has('page')) {
-    const parsedPage = parseInt(searchParams.get('page'))
-    if (isNaN(parsedPage) || parsedPage <= 0) {
-      throw Object.assign(new Error('invalid page number'), { status: 400 })
+  if (!searchParams.has('page')) {
+    let before = new Date()
+    if (searchParams.has('before')) {
+      const parsedBefore = new Date(searchParams.get('before'))
+      if (isNaN(parsedBefore.getTime())) {
+        throw Object.assign(new Error('invalid before date'), { status: 400 })
+      }
+      before = parsedBefore
     }
-    offset = (parsedPage - 1) * size
-    page = parsedPage
+    return { before, size }
   }
 
-  let before
-  if (searchParams.has('before')) {
-    const parsedBefore = new Date(searchParams.get('before'))
-    if (isNaN(parsedBefore.getTime())) {
-      throw Object.assign(new Error('invalid before date'), { status: 400 })
-    }
-    before = parsedBefore.toISOString()
+  const page = parseInt(searchParams.get('page'))
+  if (isNaN(page) || page <= 0) {
+    throw Object.assign(new Error('invalid page number'), { status: 400 })
   }
 
-  let after
-  if (searchParams.has('after')) {
-    const parsedAfter = new Date(searchParams.get('after'))
-    if (isNaN(parsedAfter.getTime())) {
-      throw Object.assign(new Error('invalid after date'), { status: 400 })
+  let sortOrder
+  if (searchParams.has('sortOrder')) {
+    sortOrder = searchParams.get('sortOrder')
+    if (!sortableOrders.includes(sortOrder)) {
+      throw Object.assign(new Error(`Sort ordering by '${sortOrder}' is not supported. Supported sort orders are: [${sortableOrders.toString()}]`), { status: 400 })
     }
-    after = parsedAfter.toISOString()
   }
 
-  const sortBy = searchParams.get('sortBy') || 'Date'
-  const sortOrder = searchParams.get('sortOrder') || 'Desc'
-
-  if (!sortableOrders.includes(sortOrder)) {
-    throw Object.assign(new Error(`Sort ordering by '${sortOrder}' is not supported. Supported sort orders are: [${sortableOrders.toString()}]`), { status: 400 })
-  }
-
-  if (!sortableValues.includes(sortBy)) {
-    throw Object.assign(new Error(`Sorting by '${sortBy}' is not supported. Supported sort orders are: [${sortableValues.toString()}]`), { status: 400 })
+  let sortBy
+  if (searchParams.has('sortBy')) {
+    sortBy = searchParams.get('sortBy')
+    if (!sortableValues.includes(sortBy)) {
+      throw Object.assign(new Error(`Sorting by '${sortBy}' is not supported. Supported sort orders are: [${sortableValues.toString()}]`), { status: 400 })
+    }
   }
 
-  return {
-    page,
-    size,
-    offset,
-    before,
-    after,
-    sortBy,
-    sortOrder
-  }
+  return { page, size, sortBy, sortOrder }
 }