From 6691c6e80e386bc25d9608c4260855c2e9a87c40 Mon Sep 17 00:00:00 2001 From: John Spurlock Date: Thu, 9 Jun 2022 09:46:11 -0500 Subject: [PATCH] threadcap: Fallback to comments more often than Castopod, handle OrderedCollectionPage. Regen npm. --- npm/threadcap/cjs/main.js | 16 +++++++++++----- npm/threadcap/esm/main.js | 16 +++++++++++----- src/threadcap/threadcap_activitypub.ts | 6 ++++-- src/threadcap/threadcap_implementation.ts | 3 ++- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/npm/threadcap/cjs/main.js b/npm/threadcap/cjs/main.js index 8eead0c..acaac5a 100644 --- a/npm/threadcap/cjs/main.js +++ b/npm/threadcap/cjs/main.js @@ -19,7 +19,8 @@ async function findOrFetchJson(url, after, fetcher, cache, opts) { if (status !== 200) throw new Error(`Expected 200 response for ${url}, found ${status} body=${bodyText}`); const contentType = headers['content-type'] || ''; - if (!contentType.toLowerCase().includes('json')) + const foundJson = contentType.toLowerCase().includes('json') || contentType === '' && bodyText.startsWith('{"'); + if (!foundJson) throw new Error(`Expected json response for ${url}, found ${contentType} body=${bodyText}`); return JSON.parse(bodyText); } @@ -110,12 +111,17 @@ async function fetchActivityPubCommenter(attributedTo, opts) { return computeCommenter(object, updateTime); } async function fetchActivityPubReplies(id, opts) { + var _a; const { fetcher, cache, updateTime, callbacks, debug } = opts; const fetchedObject = await findOrFetchActivityPubObject(id, updateTime, fetcher, cache); const object = unwrapActivityIfNecessary(fetchedObject, id, callbacks); - const replies = object.type === 'PodcastEpisode' ? object.comments : object.replies; + const replies = object.type === 'PodcastEpisode' ? object.comments : (_a = object.replies) !== null && _a !== void 0 ? _a : object.comments; if (replies === undefined) { - const message = object.type === 'PodcastEpisode' ? `No 'comments' found on PodcastEpisode object` : `No 'replies' found on object`; + let message = object.type === 'PodcastEpisode' ? `No 'comments' found on PodcastEpisode object` : `No 'replies' found on object`; + const tryPleromaWorkaround = id.includes('/objects/'); + if (tryPleromaWorkaround) { + message += ', trying Pleroma workaround'; + } callbacks === null || callbacks === void 0 ? void 0 : callbacks.onEvent({ kind: 'warning', url: id, @@ -123,7 +129,7 @@ async function fetchActivityPubReplies(id, opts) { message, object }); - if (id.includes('/objects/')) { + if (tryPleromaWorkaround) { return await mastodonFindReplies(id, { after: updateTime, fetcher, @@ -137,7 +143,7 @@ async function fetchActivityPubReplies(id, opts) { const fetched = new Set(); if (typeof replies === 'string') { const obj = await findOrFetchActivityPubObject(replies, updateTime, fetcher, cache); - if (obj.type === 'OrderedCollection') { + if (obj.type === 'OrderedCollection' || obj.type === 'OrderedCollectionPage') { return await collectRepliesFromOrderedCollection(obj, updateTime, id, fetcher, cache, callbacks, fetched); } else { diff --git a/npm/threadcap/esm/main.js b/npm/threadcap/esm/main.js index 10770ba..a21b9b6 100644 --- a/npm/threadcap/esm/main.js +++ b/npm/threadcap/esm/main.js @@ -19,7 +19,8 @@ async function findOrFetchJson(url, after, fetcher, cache, opts) { if (status !== 200) throw new Error(`Expected 200 response for ${url}, found ${status} body=${bodyText}`); const contentType = headers['content-type'] || ''; - if (!contentType.toLowerCase().includes('json')) + const foundJson = contentType.toLowerCase().includes('json') || contentType === '' && bodyText.startsWith('{"'); + if (!foundJson) throw new Error(`Expected json response for ${url}, found ${contentType} body=${bodyText}`); return JSON.parse(bodyText); } @@ -110,12 +111,17 @@ async function fetchActivityPubCommenter(attributedTo, opts) { return computeCommenter(object, updateTime); } async function fetchActivityPubReplies(id, opts) { + var _a; const { fetcher, cache, updateTime, callbacks, debug } = opts; const fetchedObject = await findOrFetchActivityPubObject(id, updateTime, fetcher, cache); const object = unwrapActivityIfNecessary(fetchedObject, id, callbacks); - const replies = object.type === 'PodcastEpisode' ? object.comments : object.replies; + const replies = object.type === 'PodcastEpisode' ? object.comments : (_a = object.replies) !== null && _a !== void 0 ? _a : object.comments; if (replies === undefined) { - const message = object.type === 'PodcastEpisode' ? `No 'comments' found on PodcastEpisode object` : `No 'replies' found on object`; + let message = object.type === 'PodcastEpisode' ? `No 'comments' found on PodcastEpisode object` : `No 'replies' found on object`; + const tryPleromaWorkaround = id.includes('/objects/'); + if (tryPleromaWorkaround) { + message += ', trying Pleroma workaround'; + } callbacks === null || callbacks === void 0 ? void 0 : callbacks.onEvent({ kind: 'warning', url: id, @@ -123,7 +129,7 @@ async function fetchActivityPubReplies(id, opts) { message, object }); - if (id.includes('/objects/')) { + if (tryPleromaWorkaround) { return await mastodonFindReplies(id, { after: updateTime, fetcher, @@ -137,7 +143,7 @@ async function fetchActivityPubReplies(id, opts) { const fetched = new Set(); if (typeof replies === 'string') { const obj = await findOrFetchActivityPubObject(replies, updateTime, fetcher, cache); - if (obj.type === 'OrderedCollection') { + if (obj.type === 'OrderedCollection' || obj.type === 'OrderedCollectionPage') { return await collectRepliesFromOrderedCollection(obj, updateTime, id, fetcher, cache, callbacks, fetched); } else { diff --git a/src/threadcap/threadcap_activitypub.ts b/src/threadcap/threadcap_activitypub.ts index cd13542..688b9b5 100644 --- a/src/threadcap/threadcap_activitypub.ts +++ b/src/threadcap/threadcap_activitypub.ts @@ -64,7 +64,9 @@ async function fetchActivityPubReplies(id: string, opts: ProtocolUpdateMethodOpt const { fetcher, cache, updateTime, callbacks, debug } = opts; const fetchedObject = await findOrFetchActivityPubObject(id, updateTime, fetcher, cache); const object = unwrapActivityIfNecessary(fetchedObject, id, callbacks); - const replies = object.type === 'PodcastEpisode' ? object.comments : object.replies; // castopod uses 'comments' url to an OrderedCollection + // castopod uses 'comments' url to an OrderedCollection + // also found 'comments' url to an OrderedCollectionPage (no 'replies') + const replies = object.type === 'PodcastEpisode' ? object.comments : (object.replies ?? object.comments); if (replies === undefined) { let message = object.type === 'PodcastEpisode' ? `No 'comments' found on PodcastEpisode object` : `No 'replies' found on object`; const tryPleromaWorkaround = id.includes('/objects/'); @@ -84,7 +86,7 @@ async function fetchActivityPubReplies(id: string, opts: ProtocolUpdateMethodOpt const fetched = new Set(); if (typeof replies === 'string') { const obj = await findOrFetchActivityPubObject(replies, updateTime, fetcher, cache); - if (obj.type === 'OrderedCollection') { + if (obj.type === 'OrderedCollection' || obj.type === 'OrderedCollectionPage') { return await collectRepliesFromOrderedCollection(obj, updateTime, id, fetcher, cache, callbacks, fetched); } else { throw new Error(`Expected 'replies' to point to an OrderedCollection, found ${JSON.stringify(obj)}`); diff --git a/src/threadcap/threadcap_implementation.ts b/src/threadcap/threadcap_implementation.ts index d323d42..20b6216 100644 --- a/src/threadcap/threadcap_implementation.ts +++ b/src/threadcap/threadcap_implementation.ts @@ -27,7 +27,8 @@ export async function findOrFetchJson(url: string, after: Instant, fetcher: Fetc const { status, headers, bodyText } = response; if (status !== 200) throw new Error(`Expected 200 response for ${url}, found ${status} body=${bodyText}`); const contentType = headers['content-type'] || ''; - if (!contentType.toLowerCase().includes('json')) throw new Error(`Expected json response for ${url}, found ${contentType} body=${bodyText}`); + const foundJson = contentType.toLowerCase().includes('json') || contentType === '' && bodyText.startsWith('{"'); + if (!foundJson) throw new Error(`Expected json response for ${url}, found ${contentType} body=${bodyText}`); return JSON.parse(bodyText); }