Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 27 additions & 25 deletions scripts/discover-listing-oauth-metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,20 @@
* If that misses a listing, re-run without `--no-playwright` for Playwright + optional manual steps.
*/
import "dotenv/config";
import * as readline from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";

import * as schema from "#/db/schema";
import type { Database } from "#/db/index.server";
import type { AuthorizationPageProbeAttempt } from "#/lib/oauth-authorization-page-discovery";
import type { StoreListingOauthDiscoveryDetail } from "#/lib/oauth-listing-oauth-discovery.types";

import * as schema from "#/db/schema";
import {
oauthAuthorizationPageProbeTargets,
probeOAuthClientMetadataFromAuthorizationServerPage,
} from "#/lib/oauth-authorization-page-discovery";
import {
probeOAuthListingAuth,
tryResolveOAuthClientMetadataUrlFast,
} from "#/lib/oauth-listing-auth-probe";
import {
attachClientMetadataCapture,
collectLoginLinkCandidates,
Expand All @@ -51,24 +59,18 @@ import {
readAuthHintsFromBody,
tryBlueskyishIdentifierLogin,
} from "#/lib/oauth-listing-playwright-discovery";
import {
probeOAuthListingAuth,
tryResolveOAuthClientMetadataUrlFast,
} from "#/lib/oauth-listing-auth-probe";
import type { AuthorizationPageProbeAttempt } from "#/lib/oauth-authorization-page-discovery";
import {
oauthAuthorizationPageProbeTargets,
probeOAuthClientMetadataFromAuthorizationServerPage,
} from "#/lib/oauth-authorization-page-discovery";
import { sqlCategorySlugsHasProtocolBrowseableSegment } from "#/lib/product-claim-eligibility";
import { and, asc, eq, isNotNull, not, sql } from "drizzle-orm";
import { stdin as input, stdout as output } from "node:process";
import * as readline from "node:readline/promises";

import type { SkippedListingReason } from "./oauth-discovery-local-progress";

import {
defaultOAuthDiscoveryProgressPath,
isSlugInLocalSkips,
loadLocalProgress,
recordSkippedListing,
type SkippedListingReason,
} from "./oauth-discovery-local-progress";

function ts(): string {
Expand Down Expand Up @@ -125,7 +127,7 @@ async function saveLocalSkip(args: {
slug: args.listing.slug,
skippedAt: new Date().toISOString(),
reason: args.reason,
...(args.note !== undefined ? { note: args.note } : {}),
...(args.note === undefined ? {} : { note: args.note }),
});
log("local_progress_saved_skip", {
slug: args.listing.slug,
Expand All @@ -134,7 +136,7 @@ async function saveLocalSkip(args: {
});
}

async function upsertDiscovery(input: {
async function upsertDiscovery(opts: {
db: Database;
listing: ListingRow;
clientMetadataUrl: string | null;
Expand All @@ -146,22 +148,22 @@ async function upsertDiscovery(input: {
}) {
const now = new Date();
const row = {
storeListingId: input.listing.id,
slug: input.listing.slug,
clientMetadataUrl: input.clientMetadataUrl,
authMethod: input.authMethod,
resolution: input.resolution,
loginPageUrl: input.loginPageUrl,
detailJson: input.detailJson,
storeListingId: opts.listing.id,
slug: opts.listing.slug,
clientMetadataUrl: opts.clientMetadataUrl,
authMethod: opts.authMethod,
resolution: opts.resolution,
loginPageUrl: opts.loginPageUrl,
detailJson: opts.detailJson,
updatedAt: now,
};

if (input.dryRun) {
if (opts.dryRun) {
log("dry_run_upsert_discovery", row);
return;
}

await input.db
await opts.db
.insert(schema.storeListingOauthDiscovery)
.values({
...row,
Expand Down Expand Up @@ -415,7 +417,7 @@ async function runPlaywrightWithManual(args: {
}

const onProg = (event: string, data?: Record<string, unknown>) =>
log(event, { slug: args.listing.slug, ...(data ?? {}) });
log(event, { slug: args.listing.slug, ...data });

const attempts: Array<AuthorizationPageProbeAttempt> = [];
let metaUrl: string | null = null;
Expand Down
6 changes: 3 additions & 3 deletions scripts/oauth-discovery-local-progress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ export async function loadLocalProgress(
? (o.skipped as Record<string, SkippedListingRecord>)
: {};
return { version: PROGRESS_VERSION, skipped };
} catch (e: unknown) {
const err = e as NodeJS.ErrnoException;
} catch (error: unknown) {
const err = error as NodeJS.ErrnoException;
if (err.code === "ENOENT") return emptyProgress();
throw e;
throw error;
}
}

Expand Down
20 changes: 9 additions & 11 deletions scripts/sync-listing-oauth-probes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,21 +300,23 @@ async function main() {
: [storefrontUrl];

try {
let report: Awaited<ReturnType<typeof probeOAuthListingAuth>>;
let result:
| Awaited<ReturnType<typeof probeOAuthListingAuth>>
| undefined;
let lastError: unknown;
for (const probeUrl of orderedProbeUrls) {
try {
report = await probeOAuthListingAuth(probeUrl);
result = await probeOAuthListingAuth(probeUrl);
lastError = undefined;
break;
} catch (e) {
lastError = e;
} catch (error) {
lastError = error;
}
}
if (lastError !== undefined) {
if (result === undefined) {
throw lastError;
}
await persistCompleted(row, report!);
await persistCompleted(row, result);
} catch (error) {
failed++;
const message = error instanceof Error ? error.message : String(error);
Expand All @@ -329,11 +331,7 @@ async function main() {
? { discoveryClientMetadataUrl: fallbackTried }
: {}),
});
await persistError(
row,
orderedProbeUrls[orderedProbeUrls.length - 1] ?? null,
message,
);
await persistError(row, orderedProbeUrls.at(-1) ?? null, message);
}

done++;
Expand Down
2 changes: 1 addition & 1 deletion src/db/schema.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { ListingLink } from "#/lib/atproto/listing-record";
import type { DirectoryOAuthLexiconHubData } from "#/lib/oauth-lexicon-hub.types";
import type { StoreListingOauthDiscoveryDetail } from "#/lib/oauth-listing-oauth-discovery.types";
import type { OAuthAuthProbeReport } from "#/lib/oauth-listing-auth-probe";
import type { StoreListingOauthDiscoveryDetail } from "#/lib/oauth-listing-oauth-discovery.types";

import { relations, sql } from "drizzle-orm";
import {
Expand Down
4 changes: 2 additions & 2 deletions src/integrations/tanstack-query/api-admin.functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,8 @@ const getRecentListings = createServerFn({ method: "GET" })
});
});

const toIso = (d: Date | null) => (d ? d.toISOString() : null);

const getAdminOAuthUrlGaps = createServerFn({ method: "GET" })
.middleware([dbMiddleware, adminFnMiddleware])
.handler(async ({ context }) => {
Expand Down Expand Up @@ -968,8 +970,6 @@ const getAdminOAuthUrlGaps = createServerFn({ method: "GET" })
.orderBy(desc(probes.probedAt), asc(listings.slug)),
]);

const toIso = (d: Date | null) => (d ? d.toISOString() : null);

return {
missingClientMetadataUrl: missingRows.map((r) => ({
id: r.id,
Expand Down
3 changes: 1 addition & 2 deletions src/lib/oauth-authorization-page-discovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,7 @@ export async function probeOAuthClientMetadataFromAuthorizationServerPage(
const attempts: Array<AuthorizationPageProbeAttempt> = [];
let clientMetadataUrl: string | null = null;

for (let i = 0; i < targets.length; i++) {
const target = targets[i]!;
for (const [i, target] of targets.entries()) {
onProgress?.("authorization_page_probe_target_start", {
index: i + 1,
total: targets.length,
Expand Down
13 changes: 6 additions & 7 deletions src/lib/oauth-listing-playwright-discovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,11 @@ export async function readAuthHintsFromBody(page: Page): Promise<{
appPasswordMentioned: boolean;
oauthMentioned: boolean;
}> {
const text = (
await page
.locator("body")
.innerText()
.catch(() => "")
).slice(0, 150_000);
const rawText = await page
.locator("body")
.textContent()
.catch(() => "");
const text = (rawText ?? "").slice(0, 150_000);
const low = text.toLowerCase();
return {
appPasswordMentioned:
Expand Down Expand Up @@ -107,7 +106,7 @@ export async function exploreLoginEntrypoints(page: Page): Promise<{
const visited: Array<string> = [];
const candidates = await collectLoginLinkCandidates(page);
const ordered = [
...candidates.filter(oauthishCandidate),
...candidates.filter((c) => oauthishCandidate(c)),
...candidates.filter((c) => !oauthishCandidate(c)),
].slice(0, 6);

Expand Down
33 changes: 18 additions & 15 deletions src/routes/_header-layout._admin-layout.admin.oauth-url-gaps.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -54,28 +54,28 @@ const styles = stylex.create({
width: "100%",
},
table: {
width: "100%",
borderCollapse: "collapse",
fontSize: "0.875rem",
width: "100%",
},
th: {
textAlign: "left" as const,
whiteSpace: "nowrap" as const,
borderBottomColor: uiColor.border2,
borderBottomStyle: "solid",
borderBottomWidth: 1,
paddingBottom: verticalSpace.sm,
paddingTop: verticalSpace.sm,
paddingRight: horizontalSpace.md,
textAlign: "left" as const,
whiteSpace: "nowrap" as const,
paddingTop: verticalSpace.sm,
},
td: {
verticalAlign: "top" as const,
borderBottomColor: uiColor.border2,
borderBottomStyle: "solid",
borderBottomWidth: 1,
paddingBottom: verticalSpace.md,
paddingTop: verticalSpace.md,
paddingRight: horizontalSpace.md,
verticalAlign: "top" as const,
paddingTop: verticalSpace.md,
},
mono: {
fontFamily: "ui-monospace, monospace",
Expand Down Expand Up @@ -111,15 +111,14 @@ function AdminOAuthUrlGapsPage() {
<Flex direction="column" style={styles.header}>
<Heading1>OAuth client metadata gaps</Heading1>
<Body variant="secondary">
Listings with an HTTPS storefront that still need a discoverable OAuth
client-metadata URL, plus listings whose last automated probe threw an
error. Protocol directory rows and <code>at:</code> links are excluded
(same rules as the discovery script).
Listings with an HTTPS storefront that still need a discoverable
OAuth client-metadata URL, plus listings whose last automated probe
threw an error. Protocol directory rows and <code>at:</code> links
are excluded (same rules as the discovery script).
</Body>
<SmallBody variant="secondary">
Populate URLs with{" "}
<code>pnpm listing:oauth-discover-metadata</code> or sync probes; see{" "}
<code>store_listing_oauth_discovery</code> and{" "}
Populate URLs with <code>pnpm listing:oauth-discover-metadata</code>{" "}
or sync probes; see <code>store_listing_oauth_discovery</code> and{" "}
<code>store_listing_oauth_probes</code>.
</SmallBody>
</Flex>
Expand Down Expand Up @@ -199,7 +198,9 @@ function GapTable({
>
{row.name}
</ProductLink>
<span {...stylex.props(styles.mono)}>{row.slug}</span>
<span {...stylex.props(styles.mono)}>
{row.slug}
</span>
</Flex>
</td>
<td {...stylex.props(styles.td)}>
Expand Down Expand Up @@ -261,7 +262,9 @@ function GapTable({
>
{row.name}
</ProductLink>
<span {...stylex.props(styles.mono)}>{row.slug}</span>
<span {...stylex.props(styles.mono)}>
{row.slug}
</span>
</Flex>
</td>
<td {...stylex.props(styles.td)}>
Expand Down
Loading