diff --git a/packages/api/src/errors.js b/packages/api/src/errors.js index c904c38..4d5fbea 100644 --- a/packages/api/src/errors.js +++ b/packages/api/src/errors.js @@ -89,6 +89,19 @@ export class TokenNotFoundError extends HTTPError { } TokenNotFoundError.CODE = 'ERROR_TOKEN_NOT_FOUND' +export class UrlNotFoundError extends Error { + /** + * @param {string} message + */ + constructor(message = 'URL Not Found') { + super(message) + this.name = 'UrlNotFoundError' + this.status = 404 + this.code = UrlNotFoundError.CODE + } +} +UrlNotFoundError.CODE = 'ERROR_URL_NOT_FOUND' + export class UnrecognisedTokenError extends HTTPError { constructor(msg = 'Could not parse provided API token') { super(msg, 401) diff --git a/packages/api/src/index.js b/packages/api/src/index.js index 18ffde0..da0ab9f 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -5,6 +5,7 @@ import { Router } from 'itty-router' import { withAuth } from './auth.js' import { permaCachePost, + permaCacheGet, permaCacheListGet, permaCacheAccountGet, permaCacheDelete, @@ -32,6 +33,7 @@ router .get('/perma-cache/status', (request) => { return Response.redirect(request.url.replace('status', 'account'), 302) }) + .get('/perma-cache/:url', permaCacheGet) .delete('/perma-cache/:url', auth['🔒'](permaCacheDelete)) /** diff --git a/packages/api/src/perma-cache/delete.js b/packages/api/src/perma-cache/delete.js index 7b6925c..8f2a8d4 100644 --- a/packages/api/src/perma-cache/delete.js +++ b/packages/api/src/perma-cache/delete.js @@ -1,7 +1,6 @@ /* eslint-env serviceworker, browser */ -// TODO: Move to separate file -import { getSourceUrl, getNormalizedUrl } from './post.js' +import { getSourceUrl, getNormalizedUrl } from '../utils/url.js' import { JSONResponse } from '../utils/json-response.js' /** * @typedef {import('../env').Env} Env diff --git a/packages/api/src/perma-cache/get.js b/packages/api/src/perma-cache/get.js index 48a0d21..2b723e1 100644 --- a/packages/api/src/perma-cache/get.js +++ b/packages/api/src/perma-cache/get.js @@ -1,96 +1,28 @@ /* eslint-env serviceworker, browser */ /* global Response */ -import { JSONResponse } from '../utils/json-response.js' +import { getSourceUrl, getNormalizedUrl } from '../utils/url.js' +import { UrlNotFoundError } from '../errors.js' /** * @typedef {import('../env').Env} Env */ /** - * Handle perma-cache put request + * Handle perma-cache get request * * @param {Request} request * @param {Env} env */ -export async function permaCacheListGet(request, env) { - const requestUrl = new URL(request.url) - const { searchParams } = requestUrl - const { size, page, sort, order } = parseSearchParams(searchParams) - - const entries = await env.db.listPermaCache(request.auth.user.id, { - size, - page, - sort, - order, - }) - - // Get next page link - const headers = - entries.length === size - ? { - Link: `<${requestUrl.pathname}?size=${size}&page=${ - page + 1 - }>; rel="next"`, - } - : undefined - return new JSONResponse(entries, { headers }) -} - -/** - * @param {URLSearchParams} searchParams - */ -function parseSearchParams(searchParams) { - // Parse size parameter - let size = 25 - if (searchParams.has('size')) { - const parsedSize = parseInt(searchParams.get('size')) - if (isNaN(parsedSize) || parsedSize <= 0 || parsedSize > 1000) { - throw Object.assign(new Error('invalid page size'), { status: 400 }) - } - size = parsedSize +export async function permaCacheGet(request, env) { + const sourceUrl = getSourceUrl(request, env) + const normalizedUrl = getNormalizedUrl(sourceUrl, env) + const r2Key = normalizedUrl.toString() + + const r2Object = await env.SUPERHOT.get(r2Key) + if (r2Object) { + return new Response(r2Object.body) } - // Parse cursor parameter - let page = 0 - if (searchParams.has('page')) { - const parsedPage = parseInt(searchParams.get('page')) - if (isNaN(parsedPage) || parsedPage <= 0) { - throw Object.assign(new Error('invalid page number'), { status: 400 }) - } - page = parsedPage - } - - // Parse sort parameter - let sort = 'date' - if (searchParams.has('sort')) { - const parsedSort = searchParams.get('sort') - if (parsedSort !== 'date' && parsedSort !== 'size') { - throw Object.assign( - new Error('invalid list sort, either "date" or "size"'), - { status: 400 } - ) - } - sort = parsedSort - } - - // Parse order parameter - let order = 'asc' - if (searchParams.has('order')) { - const parsedOrder = searchParams.get('order') - if (parsedOrder !== 'asc' && parsedOrder !== 'desc') { - throw Object.assign( - new Error('invalid list sort order, either "asc" or "desc"'), - { status: 400 } - ) - } - sort = parsedOrder - } - - return { - size, - page, - sort, - order, - } + throw new UrlNotFoundError() } diff --git a/packages/api/src/perma-cache/index.js b/packages/api/src/perma-cache/index.js index a6fdf92..6bb85c8 100644 --- a/packages/api/src/perma-cache/index.js +++ b/packages/api/src/perma-cache/index.js @@ -1,4 +1,5 @@ export { permaCachePost } from './post.js' -export { permaCacheListGet } from './get.js' +export { permaCacheGet } from './get.js' export { permaCacheAccountGet } from './account.js' +export { permaCacheListGet } from './list.js' export { permaCacheDelete } from './delete.js' diff --git a/packages/api/src/perma-cache/list.js b/packages/api/src/perma-cache/list.js new file mode 100644 index 0000000..f966652 --- /dev/null +++ b/packages/api/src/perma-cache/list.js @@ -0,0 +1,96 @@ +/* eslint-env serviceworker, browser */ +/* global Response */ + +import { JSONResponse } from '../utils/json-response.js' + +/** + * @typedef {import('../env').Env} Env + */ + +/** + * Handle perma-cache list get request + * + * @param {Request} request + * @param {Env} env + */ +export async function permaCacheListGet(request, env) { + const requestUrl = new URL(request.url) + const { searchParams } = requestUrl + const { size, page, sort, order } = parseSearchParams(searchParams) + + const entries = await env.db.listPermaCache(request.auth.user.id, { + size, + page, + sort, + order, + }) + + // Get next page link + const headers = + entries.length === size + ? { + Link: `<${requestUrl.pathname}?size=${size}&page=${ + page + 1 + }>; rel="next"`, + } + : undefined + return new JSONResponse(entries, { headers }) +} + +/** + * @param {URLSearchParams} searchParams + */ +function parseSearchParams(searchParams) { + // Parse size parameter + let size = 25 + if (searchParams.has('size')) { + const parsedSize = parseInt(searchParams.get('size')) + if (isNaN(parsedSize) || parsedSize <= 0 || parsedSize > 1000) { + throw Object.assign(new Error('invalid page size'), { status: 400 }) + } + size = parsedSize + } + + // Parse cursor parameter + let page = 0 + if (searchParams.has('page')) { + const parsedPage = parseInt(searchParams.get('page')) + if (isNaN(parsedPage) || parsedPage <= 0) { + throw Object.assign(new Error('invalid page number'), { status: 400 }) + } + page = parsedPage + } + + // Parse sort parameter + let sort = 'date' + if (searchParams.has('sort')) { + const parsedSort = searchParams.get('sort') + if (parsedSort !== 'date' && parsedSort !== 'size') { + throw Object.assign( + new Error('invalid list sort, either "date" or "size"'), + { status: 400 } + ) + } + sort = parsedSort + } + + // Parse order parameter + let order = 'asc' + if (searchParams.has('order')) { + const parsedOrder = searchParams.get('order') + if (parsedOrder !== 'asc' && parsedOrder !== 'desc') { + throw Object.assign( + new Error('invalid list sort order, either "asc" or "desc"'), + { status: 400 } + ) + } + sort = parsedOrder + } + + return { + size, + page, + sort, + order, + } +} diff --git a/packages/api/src/perma-cache/post.js b/packages/api/src/perma-cache/post.js index 3fb4eda..e65093e 100644 --- a/packages/api/src/perma-cache/post.js +++ b/packages/api/src/perma-cache/post.js @@ -1,18 +1,10 @@ /* eslint-env serviceworker, browser */ /* global Response */ -import { - MAX_ALLOWED_URL_LENGTH, - INVALID_PERMA_CACHE_CACHE_CONTROL_DIRECTIVES, -} from '../constants.js' -import { - InvalidUrlError, - TimeoutError, - HTTPError, - ExpectationFailedError, -} from '../errors.js' +import { INVALID_PERMA_CACHE_CACHE_CONTROL_DIRECTIVES } from '../constants.js' +import { TimeoutError, HTTPError, ExpectationFailedError } from '../errors.js' import { JSONResponse } from '../utils/json-response.js' -import { normalizeCid } from '../utils/cid.js' +import { getSourceUrl, getNormalizedUrl } from '../utils/url.js' /** * @typedef {import('../env').Env} Env @@ -114,85 +106,6 @@ async function getResponse(request, env, url) { return response } -/** - * Verify if provided url is a valid nftstorage.link URL - * Returns subdomain format. - * - * @param {Request} request - * @param {Env} env - */ -export function getSourceUrl(request, env) { - let candidateUrl - try { - candidateUrl = new URL(decodeURIComponent(request.params.url)) - } catch (err) { - throw new InvalidUrlError( - `invalid URL provided: ${request.params.url}: ${err.message}` - ) - } - - const urlString = candidateUrl.toString() - if (urlString.length > MAX_ALLOWED_URL_LENGTH) { - throw new InvalidUrlError( - `invalid URL provided: ${request.params.url}: maximum allowed length or URL is ${MAX_ALLOWED_URL_LENGTH}` - ) - } - if (!urlString.includes(env.GATEWAY_DOMAIN)) { - throw new InvalidUrlError( - `invalid URL provided: ${urlString}: not nftstorage.link URL` - ) - } - - return candidateUrl -} - -/** - * Verify if candidate url has IPFS path or IPFS subdomain, returning subdomain format. - * - * @param {URL} candidateUrl - * @param {Env} env - */ -export function getNormalizedUrl(candidateUrl, env) { - // Verify if IPFS path resolution URL - const ipfsPathParts = candidateUrl.pathname.split('/ipfs/') - if (ipfsPathParts.length > 1) { - const pathParts = ipfsPathParts[1].split(/\/(.*)/s) - const cid = getCid(pathParts[0]) - - // Parse path + query params - const path = pathParts[1] ? `/${pathParts[1]}` : '' - const queryParamsCandidate = candidateUrl.searchParams.toString() - const queryParams = queryParamsCandidate.length - ? `?${queryParamsCandidate}` - : '' - - return new URL( - `${candidateUrl.protocol}//${cid}.ipfs.${env.GATEWAY_DOMAIN}${path}${queryParams}` - ) - } - - // Verify if subdomain resolution URL - const subdomainParts = candidateUrl.hostname.split('.ipfs.') - if (subdomainParts.length <= 1) { - throw new InvalidUrlError( - `invalid URL provided: ${candidateUrl}: not subdomain nor ipfs path available` - ) - } - - return candidateUrl -} - -/** - * @param {string} candidateCid - */ -function getCid(candidateCid) { - try { - return normalizeCid(candidateCid) - } catch (err) { - throw new InvalidUrlError(`invalid CID: ${candidateCid}: ${err.message}`) - } -} - /** * Validates cache control header to verify if we should perma cache the response. * Based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control diff --git a/packages/api/src/utils/url.js b/packages/api/src/utils/url.js new file mode 100644 index 0000000..c3d6a90 --- /dev/null +++ b/packages/api/src/utils/url.js @@ -0,0 +1,82 @@ +import { MAX_ALLOWED_URL_LENGTH } from '../constants.js' +import { InvalidUrlError } from '../errors.js' + +import { normalizeCid } from './cid.js' + +/** + * Parses provided URL and verifes if is a valid nftstorage.link URL + * + * @param {Request} request + * @param {Env} env + */ +export function getSourceUrl(request, env) { + let candidateUrl + try { + candidateUrl = new URL(decodeURIComponent(request.params.url)) + } catch (err) { + throw new InvalidUrlError( + `invalid URL provided: ${request.params.url}: ${err.message}` + ) + } + + const urlString = candidateUrl.toString() + if (urlString.length > MAX_ALLOWED_URL_LENGTH) { + throw new InvalidUrlError( + `invalid URL provided: ${request.params.url}: maximum allowed length or URL is ${MAX_ALLOWED_URL_LENGTH}` + ) + } + if (!urlString.includes(env.GATEWAY_DOMAIN)) { + throw new InvalidUrlError( + `invalid URL provided: ${urlString}: not nftstorage.link URL` + ) + } + + return candidateUrl +} + +/** + * Verify if candidate url has IPFS path or IPFS subdomain, returning subdomain format. + * + * @param {URL} candidateUrl + * @param {Env} env + */ +export function getNormalizedUrl(candidateUrl, env) { + // Verify if IPFS path resolution URL + const ipfsPathParts = candidateUrl.pathname.split('/ipfs/') + if (ipfsPathParts.length > 1) { + const pathParts = ipfsPathParts[1].split(/\/(.*)/s) + const cid = getCid(pathParts[0]) + + // Parse path + query params + const path = pathParts[1] ? `/${pathParts[1]}` : '' + const queryParamsCandidate = candidateUrl.searchParams.toString() + const queryParams = queryParamsCandidate.length + ? `?${queryParamsCandidate}` + : '' + + return new URL( + `${candidateUrl.protocol}//${cid}.ipfs.${env.GATEWAY_DOMAIN}${path}${queryParams}` + ) + } + + // Verify if subdomain resolution URL + const subdomainParts = candidateUrl.hostname.split('.ipfs.') + if (subdomainParts.length <= 1) { + throw new InvalidUrlError( + `invalid URL provided: ${candidateUrl}: not subdomain nor ipfs path available` + ) + } + + return candidateUrl +} + +/** + * @param {string} candidateCid + */ +function getCid(candidateCid) { + try { + return normalizeCid(candidateCid) + } catch (err) { + throw new InvalidUrlError(`invalid CID: ${candidateCid}: ${err.message}`) + } +} diff --git a/packages/api/test/perma-cache-get.spec.js b/packages/api/test/perma-cache-get.spec.js index f4c2086..397d546 100644 --- a/packages/api/test/perma-cache-get.spec.js +++ b/packages/api/test/perma-cache-get.spec.js @@ -1,5 +1,4 @@ import test from 'ava' -import pMap from 'p-map' import { getMiniflare } from './scripts/utils.js' import { createTestUser } from './scripts/helpers.js' @@ -17,121 +16,45 @@ test.beforeEach(async (t) => { } }) -// PUT /perma-cache -test('Gets empty list when there were no perma-cached objects previously added', async (t) => { +test('Gets content from perma cache by URL', async (t) => { const { mf, user } = t.context - const response = await mf.dispatchFetch( - 'https://localhost:8788/perma-cache', - { - method: 'GET', - headers: { Authorization: `Bearer ${user.token}` }, - } - ) - t.is(response.status, 200) - const entries = await response.json() - t.is(entries.length, 0) -}) + const url = + 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq' + const gatewayTxtResponse = 'Hello nft.storage! 😎' -test('Gets list when there were perma-cached objects previously added', async (t) => { - const { mf, user } = t.context + // Post URL content to perma cache + const responsePost = await mf.dispatchFetch(getPermaCachePutUrl(url), { + method: 'POST', + headers: { Authorization: `Bearer ${user.token}` }, + }) + t.is(responsePost.status, 200) - // Perma cache URLs - const urls = [ - 'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:9081?download=true', - 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq', - 'http://localhost:9081/ipfs/bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq/path', - ] - await pMap( - urls, - async (url) => { - const putResponse = await mf.dispatchFetch(getPermaCachePutUrl(url), { - method: 'POST', - headers: { Authorization: `Bearer ${user.token}` }, - }) - t.is(putResponse.status, 200) - }, - { concurrency: 1 } - ) + // GET URL content from perma cache + const { normalizedUrl } = getParsedUrl(url) - // Get URLs - const listResponse = await mf.dispatchFetch( - 'https://localhost:8788/perma-cache', + const responseGet = await mf.dispatchFetch( + getPermaCachePutUrl(normalizedUrl), { method: 'GET', - headers: { Authorization: `Bearer ${user.token}` }, } ) - t.is(listResponse.status, 200) - - const entries = await listResponse.json() - t.is(entries.length, urls.length) - - // Validate content - validateList(t, urls, entries) + t.is(responseGet.status, 200) + t.deepEqual(await responseGet.text(), gatewayTxtResponse) }) -test('Can paginate list', async (t) => { - const { mf, user } = t.context - - // Perma-cache URLs - const urls = [ - 'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:9081?download=true', - 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq', - 'http://localhost:9081/ipfs/bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq/path', - ] - await pMap( - urls, - async (url) => { - const putResponse = await mf.dispatchFetch(getPermaCachePutUrl(url), { - method: 'POST', - headers: { Authorization: `Bearer ${user.token}` }, - }) - t.is(putResponse.status, 200) - }, - { concurrency: 1 } - ) - - const pages = [] - const size = 2 +test('Gets 404 response from perma cache by URL when url not perma cached', async (t) => { + const { mf } = t.context + const url = + 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq' - let nextPageLink = `https://localhost:8788/perma-cache?size=${size}` - do { - const listResponse = await mf.dispatchFetch(nextPageLink, { + // GET URL content from perma cache + const { normalizedUrl } = getParsedUrl(url) + const responseGet = await mf.dispatchFetch( + getPermaCachePutUrl(normalizedUrl), + { method: 'GET', - headers: { Authorization: `Bearer ${user.token}` }, - }) - t.is(listResponse.status, 200) - - const entries = await listResponse.json() - pages.push(entries) - - const link = listResponse.headers.get('link') - if (link) { - nextPageLink = `https://localhost:8788${link - .replace('<', '') - .replace('>; rel="next"', '')}` - } else { - nextPageLink = undefined } - } while (nextPageLink) - - t.is(pages.length, Math.round(urls.length / size)) - - const allPages = pages.flat() - t.is(allPages.length, urls.length) - - // Validate content - validateList(t, urls, allPages) + ) + t.is(responseGet.status, 404) }) - -const validateList = (t, urls, entries) => { - urls.forEach((url) => { - const { sourceUrl } = getParsedUrl(url) - - const target = entries.find((e) => sourceUrl === e.url) - t.is(sourceUrl, target.url) - t.truthy(target.insertedAt) - t.truthy(target.size) - }) -} diff --git a/packages/api/test/perma-cache-list.spec.js b/packages/api/test/perma-cache-list.spec.js new file mode 100644 index 0000000..f4c2086 --- /dev/null +++ b/packages/api/test/perma-cache-list.spec.js @@ -0,0 +1,137 @@ +import test from 'ava' +import pMap from 'p-map' + +import { getMiniflare } from './scripts/utils.js' +import { createTestUser } from './scripts/helpers.js' +import { getParsedUrl, getPermaCachePutUrl } from './utils.js' + +test.beforeEach(async (t) => { + const user = await createTestUser({ + grantRequiredTags: true, + }) + + // Create a new Miniflare environment for each test + t.context = { + mf: getMiniflare(), + user, + } +}) + +// PUT /perma-cache +test('Gets empty list when there were no perma-cached objects previously added', async (t) => { + const { mf, user } = t.context + const response = await mf.dispatchFetch( + 'https://localhost:8788/perma-cache', + { + method: 'GET', + headers: { Authorization: `Bearer ${user.token}` }, + } + ) + t.is(response.status, 200) + + const entries = await response.json() + t.is(entries.length, 0) +}) + +test('Gets list when there were perma-cached objects previously added', async (t) => { + const { mf, user } = t.context + + // Perma cache URLs + const urls = [ + 'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:9081?download=true', + 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq', + 'http://localhost:9081/ipfs/bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq/path', + ] + await pMap( + urls, + async (url) => { + const putResponse = await mf.dispatchFetch(getPermaCachePutUrl(url), { + method: 'POST', + headers: { Authorization: `Bearer ${user.token}` }, + }) + t.is(putResponse.status, 200) + }, + { concurrency: 1 } + ) + + // Get URLs + const listResponse = await mf.dispatchFetch( + 'https://localhost:8788/perma-cache', + { + method: 'GET', + headers: { Authorization: `Bearer ${user.token}` }, + } + ) + t.is(listResponse.status, 200) + + const entries = await listResponse.json() + t.is(entries.length, urls.length) + + // Validate content + validateList(t, urls, entries) +}) + +test('Can paginate list', async (t) => { + const { mf, user } = t.context + + // Perma-cache URLs + const urls = [ + 'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:9081?download=true', + 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq', + 'http://localhost:9081/ipfs/bafybeih74zqc6kamjpruyra4e4pblnwdpickrvk4hvturisbtveghflovq/path', + ] + await pMap( + urls, + async (url) => { + const putResponse = await mf.dispatchFetch(getPermaCachePutUrl(url), { + method: 'POST', + headers: { Authorization: `Bearer ${user.token}` }, + }) + t.is(putResponse.status, 200) + }, + { concurrency: 1 } + ) + + const pages = [] + const size = 2 + + let nextPageLink = `https://localhost:8788/perma-cache?size=${size}` + do { + const listResponse = await mf.dispatchFetch(nextPageLink, { + method: 'GET', + headers: { Authorization: `Bearer ${user.token}` }, + }) + t.is(listResponse.status, 200) + + const entries = await listResponse.json() + pages.push(entries) + + const link = listResponse.headers.get('link') + if (link) { + nextPageLink = `https://localhost:8788${link + .replace('<', '') + .replace('>; rel="next"', '')}` + } else { + nextPageLink = undefined + } + } while (nextPageLink) + + t.is(pages.length, Math.round(urls.length / size)) + + const allPages = pages.flat() + t.is(allPages.length, urls.length) + + // Validate content + validateList(t, urls, allPages) +}) + +const validateList = (t, urls, entries) => { + urls.forEach((url) => { + const { sourceUrl } = getParsedUrl(url) + + const target = entries.find((e) => sourceUrl === e.url) + t.is(sourceUrl, target.url) + t.truthy(target.insertedAt) + t.truthy(target.size) + }) +} diff --git a/packages/edge-gateway/src/env.js b/packages/edge-gateway/src/env.js index 1fd9261..4bd3d38 100644 --- a/packages/edge-gateway/src/env.js +++ b/packages/edge-gateway/src/env.js @@ -7,6 +7,7 @@ import { Logging } from './logs.js' * @typedef {Object} EnvInput * @property {string} IPFS_GATEWAYS * @property {string} GATEWAY_HOSTNAME + * @property {string} EDGE_GATEWAY_API_URL * @property {string} VERSION * @property {string} SENTRY_RELEASE * @property {string} COMMITHASH @@ -21,7 +22,7 @@ import { Logging } from './logs.js' * @property {Object} CIDSTRACKER * @property {Object} GATEWAYREDIRECTCOUNTER * @property {KVNamespace} DENYLIST - * @property {R2Bucket} SUPERHOT + * @property {CFService} API * * @typedef {Object} EnvTransformed * @property {string} IPFS_GATEWAY_HOSTNAME @@ -99,6 +100,8 @@ function getSentry(request, env, ctx) { /** * From: https://github.com/cloudflare/workers-types * + * @typedef {{ fetch: fetch }} CFService + * * @typedef {{ * toString(): string * equals(other: DurableObjectId): boolean @@ -119,19 +122,4 @@ function getSentry(request, env, ctx) { * }} DurableObjectStub * * @typedef {{ get: (key: string) => Promise }} KVNamespace - * - * @typedef {Object} R2PutOptions - * @property {Headers} [httpMetadata] - * @property {Record} [customMetadata] - * - * @typedef {Object} R2Object - * @property {Date} uploaded - * @property {Headers} [httpMetadata] - * @property {Record} [customMetadata] - * - * @typedef {Object} R2Bucket - * @property {(key: string) => Promise} head - * @property {(key: string) => Promise} get - * @property {(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null, options?: R2PutOptions) => Promise} put - * @property {(key: string) => Promise} delete */ diff --git a/packages/edge-gateway/src/gateway.js b/packages/edge-gateway/src/gateway.js index 68bae63..187ac1d 100644 --- a/packages/edge-gateway/src/gateway.js +++ b/packages/edge-gateway/src/gateway.js @@ -272,7 +272,7 @@ async function cdnResolution(request, env, cache) { try { const res = await pAny( - [cache.match(request.url), env.SUPERHOT.get(request.url)], + [cache.match(request.url), getFromPermaCache(request, env)], { filter: (res) => !!res, } @@ -286,6 +286,30 @@ async function cdnResolution(request, env, cache) { } } +/** + * Get from Perma Cache route. + * + * @param {Request} request + * @param {Env} env + * @return {Promise} + */ +async function getFromPermaCache(request, env) { + const req = await env.API.fetch( + new URL( + `/perma-cache/${encodeURIComponent(request.url)}`, + env.EDGE_GATEWAY_API_URL + ).toString(), + { + headers: request.headers, + } + ) + if (req.ok) { + return req + } + + return undefined +} + /** * Store metrics for winner gateway response * diff --git a/packages/edge-gateway/test/cdn.spec.js b/packages/edge-gateway/test/cdn.spec.js index 3154d26..0d6a205 100644 --- a/packages/edge-gateway/test/cdn.spec.js +++ b/packages/edge-gateway/test/cdn.spec.js @@ -28,31 +28,12 @@ test.skip('Caches content on resolve', async (t) => { test('Get content from Perma cache if existing', async (t) => { // Should go through Perma cache bucket - t.plan(2) const { mf } = t.context const url = - 'https://bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o23urcs4eige.ipfs.localhost:8787/' - - const bindings = await mf.getBindings() - const r2Bucket = bindings.SUPERHOT - const wrappedBucket = Object.assign( - { ...r2Bucket }, - { - get: async (key) => { - const r2Object = await r2Bucket.get(key) - t.truthy(r2Object) - - return r2Object - }, - } - ) - await mf.setOptions({ - bindings: { SUPERHOT: wrappedBucket }, - }) + 'https://bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o24urcs4eige.ipfs.localhost:8787/' const content = 'Hello perma cache!' - await r2Bucket.put(url, new Response(fromString(content)).body) const response = await mf.dispatchFetch(url) await response.waitUntil() @@ -75,13 +56,8 @@ test('Fail to resolve when only-if-cached and content is not cached', async (t) test('Get content from cache when existing and only-if-cached cache control is provided', async (t) => { const { mf } = t.context const url = - 'https://bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o23urcs4eige.ipfs.localhost:8787/' - - const bindings = await mf.getBindings() - const r2Bucket = bindings.SUPERHOT - + 'https://bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o24urcs4eige.ipfs.localhost:8787/' const content = 'Hello perma cache!' - await r2Bucket.put(url, new Response(fromString(content)).body) const response = await mf.dispatchFetch(url, { headers: { 'Cache-Control': 'only-if-cached' }, @@ -92,29 +68,21 @@ test('Get content from cache when existing and only-if-cached cache control is p test('Should not get from cache if no-cache cache control header is provided', async (t) => { const url = - 'https://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:8787/' - const content = 'Hello nft.storage! 😎' + 'https://bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o24urcs4eige.ipfs.localhost:8787/' const { mf } = t.context - const bindings = await mf.getBindings() - const r2Bucket = bindings.SUPERHOT - const wrappedBucket = Object.assign( - { ...r2Bucket }, - { - get: async (key) => { - throw new Error('should not get from cache') - }, - } - ) - await mf.setOptions({ - bindings: { SUPERHOT: wrappedBucket }, - }) - // Add to cache - await r2Bucket.put(url, new Response(fromString(content)).body) - - const response = await mf.dispatchFetch(url, { - headers: { 'Cache-Control': 'no-cache' }, - }) - await response.waitUntil() - t.is(await response.text(), content) + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 2000) + + try { + await mf.dispatchFetch(url, { + headers: { 'Cache-Control': 'no-cache' }, + signal: controller.signal, + }) + throw new Error('should not resolve') + } catch (err) { + t.assert(err) + } finally { + clearTimeout(timer) + } }) diff --git a/packages/edge-gateway/test/scripts/api.js b/packages/edge-gateway/test/scripts/api.js new file mode 100644 index 0000000..d2666a5 --- /dev/null +++ b/packages/edge-gateway/test/scripts/api.js @@ -0,0 +1,19 @@ +export default { + async fetch(request) { + // TODO: hardcoded until miniflare supports R2 + if ( + request.url + .toString() + .includes('bafybeic2hr75ukgwhnasdl3sucxyfedfyp3dijq3oakzx6o24urcs4eige') + ) { + return new Response('Hello perma cache!', { + status: 200, + contentType: 'text/plain', + }) + } + + return new Response('Not Found', { + status: 404, + }) + }, +} diff --git a/packages/edge-gateway/test/utils.js b/packages/edge-gateway/test/utils.js index ae1199a..9943b48 100644 --- a/packages/edge-gateway/test/utils.js +++ b/packages/edge-gateway/test/utils.js @@ -1,4 +1,3 @@ -import { concat } from 'uint8arrays/concat' import { Miniflare } from 'miniflare' export function getMiniflare() { @@ -14,8 +13,14 @@ export function getMiniflare() { buildCommand: undefined, wranglerConfigEnv: 'test', modules: true, - bindings: { - SUPERHOT: createR2Bucket(), + mounts: { + api: { + scriptPath: './test/scripts/api.js', + modules: true, + }, + }, + serviceBindings: { + API: 'api', }, }) } @@ -31,62 +36,3 @@ export function getGatewayRateLimitsName() { export function getSummaryMetricsName() { return 'SUMMARYMETRICS' } - -function createR2Bucket() { - const bucket = new Map() - - return { - put: async (key, value, putOpts = {}) => { - let data = new Uint8Array([]) - const reader = value.getReader() - try { - while (true) { - const { done, value } = await reader.read() - if (done) { - break - } - data = concat([data, value]) - } - } finally { - reader.releaseLock() - } - - bucket.set(key, { - body: data, - httpMetadata: putOpts.httpMetadata || {}, - customMetadata: putOpts.customMetadata || {}, - }) - - return Promise.resolve({ - httpMetadata: putOpts.httpMetadata, - customMetadata: putOpts.customMetadata, - }) - }, - get: async (key) => { - const value = bucket.get(key) - if (!value) { - return undefined - } - - const response = new Response(value.body, { status: 200 }) - - return Promise.resolve( - Object.assign(response, { - httpMetadata: value.httpMetadata || {}, - customMetadata: value.customMetadata || {}, - }) - ) - }, - head: async (key) => { - const value = bucket.get(key) - if (!value) { - return undefined - } - - return Promise.resolve({ - httpMetadata: value.httpMetadata || {}, - customMetadata: value.customMetadata || {}, - }) - }, - } -} diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml index 2338ef0..c438f9d 100644 --- a/packages/edge-gateway/wrangler.toml +++ b/packages/edge-gateway/wrangler.toml @@ -32,6 +32,7 @@ kv_namespaces = [{ binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" [env.production.vars] IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://pinata.nftstorage.link\"]" GATEWAY_HOSTNAME = 'ipfs.nftstorage.link' +EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' DATABASE_URL = "https://nft-storage-pgrest-prod.herokuapp.com" DEBUG = "false" ENV = "production" @@ -44,9 +45,11 @@ bindings = [ {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] -[[env.production.r2_buckets]] -bucket_name = "super-hot" -binding = "SUPERHOT" +[[env.production.services]] +binding = "API" +type = "service" +service = "nftstorage-link-api-production" +environment = "production" # Staging! [env.staging] @@ -64,6 +67,7 @@ kv_namespaces = [{ binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" [env.staging.vars] IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://pinata.nftstorage.link\"]" GATEWAY_HOSTNAME = 'ipfs-staging.nftstorage.link' +EDGE_GATEWAY_API_URL = 'https://api-staging.nftstorage.link' DATABASE_URL = "https://nft-storage-pgrest-staging.herokuapp.com" DEBUG = "true" ENV = "staging" @@ -76,9 +80,11 @@ bindings = [ {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"} ] -[[env.staging.r2_buckets]] -bucket_name = "super-hot-staging" -binding = "SUPERHOT" +[[env.staging.services]] +binding = "API" +type = "service" +service = "nftstorage-link-api-staging" +environment = "production" # Test! [env.test] @@ -88,6 +94,7 @@ kv_namespaces = [{ binding = "DENYLIST" }] [env.test.vars] IPFS_GATEWAYS = "[\"http://127.0.0.1:9081\", \"http://localhost:9082\", \"http://localhost:9083\"]" GATEWAY_HOSTNAME = 'ipfs.localhost:8787' +EDGE_GATEWAY_API_URL = 'http://localhost:8787' DEBUG = "true" ENV = "test" @@ -108,6 +115,7 @@ kv_namespaces = [{ binding = "DENYLIST", id = "" }] [env.vsantos.vars] GATEWAY_HOSTNAME = 'ipfs.localhost:8787' IPFS_GATEWAYS = "[\"https://ipfs.io\"]" +EDGE_GATEWAY_API_URL = 'http://localhost:8787' [env.vsantos.durable_objects] bindings = [