From 63ad33b406dac38b7dc04f82fc427e128e852425 Mon Sep 17 00:00:00 2001 From: Hicks-99 <94490389+Hicks-99@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:03:40 +0200 Subject: [PATCH 1/2] feat(metadata): add Steam metadata provider (#232) --- nuxt.config.ts | 1 + .../migration.sql | 8 + prisma/models/content.prisma | 1 + server/internal/metadata/steam.ts | 1024 +++++++++++++++++ server/plugins/03.metadata-init.ts | 2 + 5 files changed, 1036 insertions(+) create mode 100644 prisma/migrations/20250918184144_add_steam_metadata/migration.sql create mode 100644 server/internal/metadata/steam.ts diff --git a/nuxt.config.ts b/nuxt.config.ts index e88def9a..8a0ebeb9 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -256,6 +256,7 @@ export default defineNuxtConfig({ "https://www.giantbomb.com", "https://images.pcgamingwiki.com", "https://images.igdb.com", + "https://*.steamstatic.com", ], }, strictTransportSecurity: false, diff --git a/prisma/migrations/20250918184144_add_steam_metadata/migration.sql b/prisma/migrations/20250918184144_add_steam_metadata/migration.sql new file mode 100644 index 00000000..5c6dbb57 --- /dev/null +++ b/prisma/migrations/20250918184144_add_steam_metadata/migration.sql @@ -0,0 +1,8 @@ +-- AlterEnum +ALTER TYPE "MetadataSource" ADD VALUE 'Steam'; + +-- DropIndex +DROP INDEX "GameTag_name_idx"; + +-- CreateIndex +CREATE INDEX "GameTag_name_idx" ON "GameTag" USING GIST ("name" gist_trgm_ops(siglen=32)); diff --git a/prisma/models/content.prisma b/prisma/models/content.prisma index 3ab28be6..1d8a0029 100644 --- a/prisma/models/content.prisma +++ b/prisma/models/content.prisma @@ -1,6 +1,7 @@ enum MetadataSource { Manual GiantBomb + Steam PCGamingWiki IGDB Metacritic diff --git a/server/internal/metadata/steam.ts b/server/internal/metadata/steam.ts new file mode 100644 index 00000000..89601411 --- /dev/null +++ b/server/internal/metadata/steam.ts @@ -0,0 +1,1024 @@ +import { MetadataSource } from "~/prisma/client/enums"; +import type { MetadataProvider } from "."; +import type { + GameMetadataSearchResult, + _FetchGameMetadataParams, + GameMetadata, + _FetchCompanyMetadataParams, + CompanyMetadata, + GameMetadataRating, +} from "./types"; +import type { TaskRunContext } from "../tasks"; +import axios from "axios"; +import * as jdenticon from "jdenticon"; + +/** + * Note: The Steam API is largely undocumented. + * Helpful resources for reverse engineering and understanding endpoints: + * - The GOAT xPaw: https://steamapi.xpaw.me/ + * - RJackson and the Team Fortress Community: https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI + * + * These community-driven resources provide valuable insights into Steam's internal APIs. + * + * Most Steam API endpoints accept a 'language' or 'l' query parameter for localization. + * Some endpoints require a cc (country code) parameter to filter region-specific game availability. + * + * There is no public known endpoint for searching companies, so we scrape the developer page instead. + * We're geussing the developer page by calling `https://store.steampowered.com/developer/{developer_name}/`. + * This as a high chance of failing, because the developer name is not always the same as the URL slug. + * Alternatively, we could use the link on a game's store page, but this redirects often to the publisher. + */ + +interface SteamItem { + appid: string; +} + +interface SteamSearchStub extends SteamItem { + name: string; + icon: string; // Ratio 1:1 + logo: string; // Ratio 8:3 +} + +interface SteamAppDetailsSmall extends SteamItem { + item_type: number; + id: number; + success: number; + visible: boolean; + name: string; + store_url_path: string; + type: number; + categories: { + supported_player_categoryids: number[]; + featured_categoryids: number[]; + controller_categoryids: number[]; + }; + basic_info: { + short_description: string; + publishers: { + name: string; + creator_clan_account_id: number; + }[]; + developers: { + name: string; + creator_clan_account_id: number; + }[]; + capsule_headline: string; + }; + release: { + steam_release_date: number; // UNIX timestamp in seconds + }; + best_purchase_option: { + packageid: number; + purchase_option_name: string; + final_price_in_cents: string; + formatted_final_price: string; + usert_can_purchase_as_gift: boolean; + hide_discount_pct_for_compliance: boolean; + included_game_count: number; + }; +} + +interface SteamAppDetailsLarge extends SteamAppDetailsSmall { + tagids: number[]; + reviews: { + summary_filtered: { + review_count: number; + percent_positive: number; + review_score: number; + review_score_label: string; + }; + summary_language_specific: { + review_count: number; + percent_positive: number; + review_score: number; + review_score_label: string; + }[]; + }; + tags: { + tagid: number; + weight: number; + }[]; + assets: { + asset_url_format: string; + main_capsule: string; + small_capsule: string; + header: string; + page_background: string; + hero_capsule: string; + library_capsule: string; + library_capsule_2x: string; + library_hero: string; + community_icon: string; + page_background_path: string; + raw_page_background: string; + }; + screenshots: { + all_ages_screenshots: { + filename: string; + ordinal: number; + }[]; + }; + full_description: string; +} + +interface SteamAppDetailsPackage { + response: { + store_items: SteamAppDetailsSmall[] | SteamAppDetailsLarge[]; + }; +} + +interface SteamTags { + tagid: number; + name: string; +} + +interface SteamTagsPackage { + response: { + version_hash: string; + tags: SteamTags[]; + }; +} + +interface SteamWebAppDetailsSmall { + type: string; + name: string; + steam_appid: number; + required_age: string; + is_free: boolean; + dlc: number[]; + detailed_description: string; + about_the_game: string; + short_description: string; + supported_languages: string; + header_image: string; + capsule_image: string; + capsule_imagev5: string; + website: string; + pc_requirements: { minimum: string; recommended: string }; + mac_requirements: { minimum: string; recommended: string }; + linux_requirements: { minimum: string; recommended: string }; + legal_notice: string; +} + +interface SteamWebAppDetailsLarge extends SteamWebAppDetailsSmall { + metacritic: { + score: number; + url: string; + }; +} + +interface SteamWebAppDetailsPackage { + [key: string]: { + success: boolean; + data: SteamWebAppDetailsSmall | SteamWebAppDetailsLarge; + }; +} + +export class SteamProvider implements MetadataProvider { + name() { + return "Steam"; + } + + source(): MetadataSource { + return MetadataSource.Steam; + } + + async search(query: string): Promise { + const response = await axios.get( + `https://steamcommunity.com/actions/SearchApps/${query}`, + ); + + if ( + response.status !== 200 || + !response.data || + response.data.length === 0 + ) { + return []; + } + + const result: GameMetadataSearchResult[] = response.data.map((item) => ({ + id: item.appid, + name: item.name, + icon: item.icon || "", + description: "", + year: 0, + })); + + const ids = response.data.map((i) => i.appid); + + const detailsResponse = await this._fetchGameDetails(ids, { + include_basic_info: true, + include_release: true, + }); + + const detailsMap = new Map(); + for (const item of detailsResponse) { + detailsMap.set(item.appid.toString(), item); + } + + for (const resItem of result) { + const details = detailsMap.get(resItem.id); + + if (!details) continue; + resItem.description = details.basic_info.short_description || ""; + + if (!details.release?.steam_release_date) continue; + const date = new Date(details.release.steam_release_date * 1000); + resItem.year = date.getFullYear(); + } + + return result; + } + + async fetchGame( + { id, publisher, developer, createObject }: _FetchGameMetadataParams, + context?: TaskRunContext, + ): Promise { + context?.logger.info(`🎮 Starting Steam metadata fetch for game ID: ${id}`); + context?.progress(0); + + context?.logger.info("📡 Fetching game details from Steam API..."); + const response = await this._fetchGameDetails([id], { + include_assets: true, + include_basic_info: true, + include_release: true, + include_screenshots: true, + include_tag_count: 100, + include_full_description: true, + include_reviews: true, + }); + + if (response.length === 0) { + context?.logger.error(`❌ No game found on Steam with ID: ${id}`); + throw new Error(`No game found on Steam with id: ${id}`); + } + + const currentGame = response[0] as SteamAppDetailsLarge; + + context?.logger.info(`✅ Found game: "${currentGame.name}" on Steam`); + context?.progress(10); + + context?.logger.info("🖼️ Processing game images and assets..."); + const { icon, cover, banner, images } = this._processImages( + currentGame, + createObject, + context, + ); + + const released = currentGame.release?.steam_release_date + ? new Date(currentGame.release.steam_release_date * 1000) + : new Date(); + + if (currentGame.release?.steam_release_date) { + context?.logger.info(`📅 Release date: ${released.toLocaleDateString()}`); + } else { + context?.logger.warn( + "⚠️ No release date found, using current date as fallback", + ); + } + + context?.progress(60); + + context?.logger.info( + `🏷️ Fetching tags from Steam (${currentGame.tagids?.length || 0} tags to process)...`, + ); + const tags = await this._getTagNames(currentGame.tagids || []); + + context?.logger.info( + `✅ Successfully fetched ${tags.length} tags: ${tags.slice(0, 5).join(", ")}${tags.length > 5 ? "..." : ""}`, + ); + context?.progress(70); + + context?.logger.info("🏢 Processing publishers and developers..."); + const publishers = []; + const publisherNames = currentGame.basic_info.publishers || []; + context?.logger.info( + `📊 Found ${publisherNames.length} publisher(s) to process`, + ); + + for (const pub of publisherNames) { + context?.logger.info(`🔍 Processing publisher: "${pub.name}"`); + const comp = await publisher(pub.name); + if (!comp) { + context?.logger.warn(`⚠️ Failed to import publisher "${pub.name}"`); + continue; + } + publishers.push(comp); + context?.logger.info(`✅ Successfully imported publisher: "${pub.name}"`); + } + + const developers = []; + const developerNames = currentGame.basic_info.developers || []; + context?.logger.info( + `📊 Found ${developerNames.length} developer(s) to process`, + ); + + for (const dev of developerNames) { + context?.logger.info(`🔍 Processing developer: "${dev.name}"`); + const comp = await developer(dev.name); + if (!comp) { + context?.logger.warn(`⚠️ Failed to import developer "${dev.name}"`); + continue; + } + developers.push(comp); + context?.logger.info(`✅ Successfully imported developer: "${dev.name}"`); + } + + context?.logger.info( + `✅ Company processing complete: ${publishers.length} publishers, ${developers.length} developers`, + ); + context?.progress(80); + + context?.logger.info("📝 Fetching detailed description and reviews..."); + const webAppDetails = (await this._getWebAppDetails( + id, + "metacritic", + )) as SteamWebAppDetailsLarge; + + const detailedDescription = + webAppDetails?.detailed_description || + webAppDetails?.about_the_game || + ""; + + let description; + if (detailedDescription) { + context?.logger.info("🔄 Converting HTML description to Markdown..."); + const converted = this._htmlToMarkdown(detailedDescription, createObject); + images.push(...converted.objects); + description = converted.markdown; + context?.logger.info( + `✅ Description converted, ${converted.objects.length} images embedded`, + ); + } else { + context?.logger.info( + "📄 Using fallback description from basic game info", + ); + description = currentGame.full_description; + } + + context?.progress(90); + + context?.logger.info("⭐ Processing review ratings..."); + const reviews = [ + { + metadataId: id, + metadataSource: MetadataSource.Steam, + mReviewCount: currentGame.reviews?.summary_filtered?.review_count || 0, + mReviewHref: `https://store.steampowered.com/app/${id}`, + mReviewRating: + (currentGame.reviews?.summary_filtered?.percent_positive || 0) / 100, + }, + ] as GameMetadataRating[]; + + const steamReviewCount = + currentGame.reviews?.summary_filtered?.review_count || 0; + const steamRating = + currentGame.reviews?.summary_filtered?.percent_positive || 0; + context?.logger.info( + `📊 Steam reviews: ${steamReviewCount} reviews, ${steamRating}% positive`, + ); + + if (webAppDetails?.metacritic) { + reviews.push({ + metadataId: id, + metadataSource: MetadataSource.Metacritic, + mReviewCount: 0, + mReviewHref: webAppDetails.metacritic.url, + mReviewRating: webAppDetails.metacritic.score / 100, + }); + context?.logger.info( + `🎯 Metacritic score: ${webAppDetails.metacritic.score}/100`, + ); + } + + context?.logger.info( + `✅ Review processing complete: ${reviews.length} rating sources found`, + ); + context?.progress(100); + + context?.logger.info("🎉 Steam metadata fetch completed successfully!"); + + return { + id: currentGame.appid.toString(), + name: currentGame.name, + shortDescription: currentGame.basic_info.short_description || "", + description, + released, + publishers, + developers, + tags, + reviews, + icon, + bannerId: banner, + coverId: cover, + images, + } as GameMetadata; + } + + async fetchCompany({ + query, + createObject, + }: _FetchCompanyMetadataParams): Promise { + const searchParams = new URLSearchParams({ + l: "english", + }); + + const response = await axios.get( + `https://store.steampowered.com/developer/${query.replaceAll(" ", "")}/?${searchParams.toString()}`, + { + maxRedirects: 0, + }, + ); + + if (response.status !== 200 || !response.data) { + return undefined; + } + + const html = response.data; + + // Extract metadata from HTML meta tags + const metadata = this._extractMetaTagsFromHtml(html); + + if (!metadata.title) { + return undefined; + } + + // Extract company name from title (format: "Steam Developer: CompanyName") + const companyName = metadata.title + .replace(/^Steam Developer:\s*/i, "") + .trim(); + + if (!companyName) { + return undefined; + } + + let logoRaw; + if (metadata.image) { + logoRaw = metadata.image; + } else { + logoRaw = jdenticon.toPng(companyName, 512); + } + + const logo = createObject(logoRaw); + + let bannerRaw; + if (metadata.banner) { + bannerRaw = metadata.banner; + } else { + bannerRaw = jdenticon.toPng(companyName, 512); + } + + const banner = createObject(bannerRaw); + + return { + id: query.replaceAll(" ", ""), + name: companyName, + shortDescription: metadata.description || "", + description: "", + logo, + banner, + website: + metadata.url || + `https://store.steampowered.com/developer/${query.replaceAll(" ", "")}`, + } as CompanyMetadata; + } + + private _extractMetaTagsFromHtml(html: string): { + title?: string; + description?: string; + image?: string; + url?: string; + banner?: string; + } { + const metadata: { + title?: string; + description?: string; + image?: string; + url?: string; + banner?: string; + } = {}; + + const title = this._extractTitle(html); + if (title) metadata.title = title; + + const description = this._extractDescription(html); + if (description) metadata.description = description; + + const image = this._extractImage(html); + if (image) metadata.image = image; + + const url = this._extractUrl(html); + if (url) metadata.url = url; + + const banner = this._extractBanner(html); + if (banner) metadata.banner = banner; + + return metadata; + } + + private _extractTitle(html: string): string | undefined { + const ogTitleRegex = + /]*>([^<]+)<\/title>/i; + + let titleMatch = ogTitleRegex.exec(html); + titleMatch ??= titleTagRegex.exec(html); + + return titleMatch ? this._decodeHtmlEntities(titleMatch[1]) : undefined; + } + + private _extractDescription(html: string): string | undefined { + const ogDescRegex = + //i; + const nameDescRegex = + //i; + + let descMatch = ogDescRegex.exec(html); + descMatch ??= nameDescRegex.exec(html); + + return descMatch ? this._decodeHtmlEntities(descMatch[1]) : undefined; + } + + private _extractImage(html: string): string | undefined { + const ogImageRegex = + /]*class\s*=\s*["'][^"']*curator_url[^"']*["'][^>]*href\s*=\s*["']https:\/\/steamcommunity\.com\/linkfilter\/\?u=([^"'&]+)["']/i; + const linkfilterRegex = + /]*href\s*=\s*["']https:\/\/steamcommunity\.com\/linkfilter\/\?u=([^"'&]+)["'][^>]*(?:target=["']_blank["']|rel=["'][^"']*["'])/i; + + let curatorUrlMatch = curatorUrlRegex.exec(html); + curatorUrlMatch ??= linkfilterRegex.exec(html); + + if (!curatorUrlMatch) return undefined; + + try { + return decodeURIComponent(curatorUrlMatch[1]); + } catch { + return curatorUrlMatch[1]; + } + } + + private _extractBanner(html: string): string | undefined { + const bannerRegex = + /background-image:\s*url\(['"]([^'"]*(?:\/clan\/\d+|\/app\/\d+|background|header)[^'"]*)\??[^'"]*['"][^}]*\)/i; + const backgroundImageRegex = + /style\s*=\s*["'][^"']*background-image:\s*url\(([^)]+)\)[^"']*/i; + + let bannerMatch = bannerRegex.exec(html); + bannerMatch ??= backgroundImageRegex.exec(html); + + if (!bannerMatch) return undefined; + + let bannerUrl = bannerMatch[1].replace(/['"]/g, ""); + // Clean up the URL + if (bannerUrl.includes("?")) { + bannerUrl = bannerUrl.split("?")[0]; + } + return bannerUrl; + } + + private _decodeHtmlEntities(text: string): string { + return text + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#x([0-9A-Fa-f]+);/g, (_, hex) => + String.fromCharCode(parseInt(hex, 16)), + ) + .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10))); + } + + private async _fetchGameDetails( + gameIds: string[], + dataRequest: object, + language = "english", + country_code = "US", + ): Promise { + const searchParams = new URLSearchParams({ + input_json: JSON.stringify({ + ids: gameIds.map((id) => ({ + appid: parseInt(id), + })), + context: { + language, + country_code, + }, + data_request: dataRequest, + }), + }); + + const request = await axios.get( + `https://api.steampowered.com/IStoreBrowseService/GetItems/v1/?${searchParams.toString()}`, + ); + + if (request.status !== 200) return []; + + const result = []; + const storeItems = request.data?.response?.store_items ?? []; + + for (const item of storeItems) { + if (item.success !== 1) continue; + result.push(item); + } + + return result; + } + + private _processImages( + game: SteamAppDetailsLarge, + createObject: (input: string | Buffer) => string, + context?: TaskRunContext, + ): { icon: string; cover: string; banner: string; images: string[] } { + const imageURLFormat = game.assets?.asset_url_format; + + context?.logger.info("🖼️ Processing game icon..."); + let iconRaw; + if (game.assets?.community_icon) { + context?.logger.info("✅ Found community icon on Steam"); + iconRaw = `https://cdn.fastly.steamstatic.com/steamcommunity/public/images/apps/${game.appid}/${game.assets.community_icon}.jpg`; + } else { + context?.logger.info("⚠️ No icon found, generating fallback icon"); + iconRaw = jdenticon.toPng(game.appid, 512); + } + + const icon = createObject(iconRaw); + context?.progress(20); + + context?.logger.info("🎨 Processing game cover art..."); + let coverRaw; + if (game.assets?.library_capsule_2x) { + context?.logger.info("✅ Found high-resolution cover art"); + coverRaw = this._getImageUrl( + game.assets.library_capsule_2x, + imageURLFormat, + ); + } else if (game.assets?.library_capsule) { + context?.logger.info("✅ Found standard resolution cover art"); + coverRaw = this._getImageUrl(game.assets.library_capsule, imageURLFormat); + } else { + context?.logger.info("⚠️ No cover art found, generating fallback cover"); + coverRaw = jdenticon.toPng(game.appid, 512); + } + + const cover = createObject(coverRaw); + context?.progress(30); + + context?.logger.info("🏞️ Processing game banner..."); + let bannerRaw; + if (game.assets?.library_hero) { + context?.logger.info("✅ Found library hero banner"); + bannerRaw = this._getImageUrl(game.assets.library_hero, imageURLFormat); + } else { + context?.logger.info("⚠️ No banner found, generating fallback banner"); + bannerRaw = jdenticon.toPng(game.appid, 512); + } + + const banner = createObject(bannerRaw); + context?.progress(40); + + const images = [cover, banner]; + const screenshotCount = game.screenshots?.all_ages_screenshots?.length || 0; + context?.logger.info(`📸 Processing ${screenshotCount} screenshots...`); + + for (const image of game.screenshots?.all_ages_screenshots || []) { + const imageUrl = this._getImageUrl(image.filename); + images.push(createObject(imageUrl)); + } + + context?.logger.info( + `✅ Image processing complete: icon, cover, banner and ${screenshotCount} screenshots`, + ); + context?.progress(50); + + return { icon, cover, banner, images }; + } + + private async _getTagNames( + tagIds: number[], + language = "english", + ): Promise { + if (tagIds.length === 0) return []; + + const searchParams = new URLSearchParams({ + language, + }); + + const request = await axios.get( + `https://api.steampowered.com/IStoreService/GetTagList/v1/?${searchParams.toString()}`, + ); + + if (request.status !== 200 || !request.data.response?.tags) return []; + + const tagMap = new Map(); + for (const tag of request.data.response.tags) { + tagMap.set(tag.tagid, tag.name); + } + + const result = []; + for (const tagId of tagIds) { + const tagName = tagMap.get(tagId); + if (!tagName) continue; + + result.push(tagName); + } + + return result; + } + + private async _getWebAppDetails( + appid: string, + dataRequest: string, // Seperated by commas + language = "english", + ): Promise { + const searchParams = new URLSearchParams({ + appids: appid, + filter: "basic," + dataRequest, + l: language, + }); + + const request = await axios.get( + `https://store.steampowered.com/api/appdetails?${searchParams.toString()}`, + ); + + if (request.status !== 200) { + return undefined; + } + + const appData = request.data[appid]?.data; + if (!appData) { + return undefined; + } + + return appData; + } + + private _getImageUrl(filename: string, format?: string): string { + if (!filename || filename.trim().length === 0) return ""; + + const url = "https://shared.fastly.steamstatic.com/store_item_assets/"; + + if (format) { + format = format.replace("${FILENAME}", filename); + return url + format; + } + + return url + filename; + } + + private _htmlToMarkdown( + html: string, + createObject: (input: string | Buffer) => string, + ): { markdown: string; objects: string[] } { + if (!html || html.trim().length === 0) return { markdown: "", objects: [] }; + + let markdown = html; + const objects: string[] = []; + const imageReplacements: { placeholder: string; imageId: string }[] = []; + + markdown = this._convertBasicHtmlElements(markdown); + + // Replace images with placheholders + markdown = markdown.replace( + /]*src\s*=\s*["']([^"']+)["'][^>]*>/gi, + (match, src) => { + const imageId = createObject(src); + objects.push(imageId); + const placeholder = `__IMG_${imageReplacements.length}__`; + imageReplacements.push({ placeholder, imageId }); + return placeholder; + }, + ); + + markdown = this._convertRemainingHtmlElements(markdown); + + markdown = this._stripHtmlTags(markdown); + + markdown = this._cleanupBasicFormatting(markdown); + + markdown = this._processImagePlaceholders(markdown, imageReplacements); + + markdown = this._finalCleanup(markdown); + + return { markdown, objects }; + } + + private _convertBasicHtmlElements(markdown: string): string { + // Remove HTML comments + markdown = markdown.replace(//g, ""); + + // Convert the bullet points and tabs to markdown list format + markdown = markdown.replace(/•\s*\t+/g, "\n- "); + + // Handle numbered enumeration (1.\t, 2.\t, etc.) + markdown = markdown.replace(/(\d+)\.\s*\t+/g, "\n$1. "); + + // Convert bold text + markdown = markdown.replace( + /<(strong|b)[^>]*>(.*?)<\/(strong|b)>/gi, + "**$2**", + ); + + // Convert headers (h1-h6) with Steam's bb_tag class + markdown = markdown.replace( + /]*>(.*?)<\/h[1-6]>/gi, + (_, level, content) => { + const headerLevel = "#".repeat(parseInt(level)); + const cleanContent = this._stripHtmlTags(content).trim(); + return cleanContent ? `\n\n${headerLevel} ${cleanContent}\n\n` : ""; + }, + ); + + return markdown; + } + + private _convertRemainingHtmlElements(markdown: string): string { + // Convert paragraphs with Steam's bb_paragraph class + markdown = markdown.replace( + /]*>(.*?)<\/p>/gi, + (_, content) => { + const cleanContent = this._stripHtmlTags(content).trim(); + return cleanContent ? `${cleanContent}` : ""; + }, + ); + + // Convert unordered lists with Steam's bb_ul class + markdown = markdown.replace( + /]*>([\s\S]*?)<\/ul>/gi, + (_, content) => { + const listItems = content.match(/]*>([\s\S]*?)<\/li>/gi) || []; + const markdownItems = listItems + .map((item: string) => { + const cleanItem = item.replace(/]*>([\s\S]*?)<\/li>/i, "$1"); + const cleanContent = this._stripHtmlTags(cleanItem).trim(); + return cleanContent ? `- ${cleanContent}` : ""; + }) + .filter(Boolean); + return markdownItems.length > 0 ? `${markdownItems.join("\n")}\n` : ""; + }, + ); + + // Convert ordered lists with Steam's bb_ol class + markdown = markdown.replace( + /]*>([\s\S]*?)<\/ol>/gi, + (_, content) => { + const listItems = content.match(/]*>([\s\S]*?)<\/li>/gi) || []; + const markdownItems = listItems + .map((item: string, index: number) => { + const cleanItem = item.replace(/]*>([\s\S]*?)<\/li>/i, "$1"); + const cleanContent = this._stripHtmlTags(cleanItem).trim(); + return cleanContent ? `${index + 1}. ${cleanContent}` : ""; + }) + .filter(Boolean); + return markdownItems.length > 0 ? `${markdownItems.join("\n")}\n` : ""; + }, + ); + + // Convert line breaks + markdown = markdown.replace(//gi, "\n"); + + // Convert italic text with and tags + markdown = markdown.replace(/<(em|i)[^>]*>(.*?)<\/(em|i)>/gi, "*$2*"); + + // Convert underlined text + markdown = markdown.replace(/]*>(.*?)<\/u>/gi, "_$1_"); + + // Convert links + markdown = markdown.replace( + /]*href\s*=\s*["']([^"']+)["'][^>]*>(.*?)<\/a>/gi, + "[$2]($1)", + ); + + // Convert divs to line breaks (common in Steam descriptions) + markdown = markdown.replace(/]*>(.*?)<\/div>/gi, "$1\n"); + + // Handle span tags with bb_img_ctn class (Steam image containers) + markdown = markdown.replace( + /]*>(.*?)<\/span>/gi, + "$1", + ); + + return markdown; + } + + private _cleanupBasicFormatting(markdown: string): string { + // Clean up spaces before newlines + markdown = markdown.replace(/ +\n/g, "\n"); + + // Clean up excessive spacing around punctuation + markdown = markdown.replace(/\s+([.,!?;:])/g, "$1"); + + return markdown; + } + + private _processImagePlaceholders( + markdown: string, + imageReplacements: { placeholder: string; imageId: string }[], + ): string { + const lines = markdown.split("\n"); + const processedLines: string[] = []; + + for (const line of lines) { + const replacedLines = this._replacePlaceholdersInLine( + line, + imageReplacements, + ); + processedLines.push(...replacedLines); + } + + return processedLines.join("\n"); + } + + private _replacePlaceholdersInLine( + line: string, + imageReplacements: { placeholder: string; imageId: string }[], + ): string[] { + const currentLine = line; + const results: string[] = []; + + // Find all placeholders + const placeholdersInLine = imageReplacements.filter(({ placeholder }) => + currentLine.includes(placeholder), + ); + + if (placeholdersInLine.length === 0) { + return [line]; + } + + // Sort placeholders by their position + placeholdersInLine.sort( + (a, b) => + currentLine.indexOf(a.placeholder) - currentLine.indexOf(b.placeholder), + ); + + let lastIndex = 0; + + for (const { placeholder, imageId } of placeholdersInLine) { + const placeholderIndex = currentLine.indexOf(placeholder, lastIndex); + + if (placeholderIndex === -1) continue; + + // Add text before the placeholder (if any) + const beforeText = currentLine.substring(lastIndex, placeholderIndex); + if (beforeText.trim()) { + results.push(beforeText.trim()); + results.push(""); // Empty line before image + } + + results.push(`![](/api/v1/object/${imageId})`); + + lastIndex = placeholderIndex + placeholder.length; + } + + // Add any remaining text after the last placeholder + const afterText = currentLine.substring(lastIndex); + if (afterText.trim()) { + results.push(""); // Empty line after image + results.push(afterText.trim()); + } + + // If we only have images and no text, return just the images + if ( + results.every( + (line) => line === "" || line.startsWith("![](/api/v1/object/"), + ) + ) { + return results.filter((line) => line !== ""); + } + + return results; + } + + private _finalCleanup(markdown: string): string { + // Clean up multiple consecutive newlines + markdown = markdown.replace(/\n{3,}/g, "\n\n"); + + markdown = markdown.trim(); + + return markdown; + } + + private _stripHtmlTags(html: string): string { + return html + .replace(/<[^>]*>/g, "") + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'"); + } +} diff --git a/server/plugins/03.metadata-init.ts b/server/plugins/03.metadata-init.ts index c9f2616a..7a289ca4 100644 --- a/server/plugins/03.metadata-init.ts +++ b/server/plugins/03.metadata-init.ts @@ -5,11 +5,13 @@ import { GiantBombProvider } from "../internal/metadata/giantbomb"; import { IGDBProvider } from "../internal/metadata/igdb"; import { ManualMetadataProvider } from "../internal/metadata/manual"; import { PCGamingWikiProvider } from "../internal/metadata/pcgamingwiki"; +import { SteamProvider } from "../internal/metadata/steam"; import { logger } from "~/server/internal/logging"; export default defineNitroPlugin(async (_nitro) => { const metadataProviders = [ GiantBombProvider, + SteamProvider, PCGamingWikiProvider, IGDBProvider, ]; From 99bd5a1ac81f010b0423ec7c72279060c4dbcda9 Mon Sep 17 00:00:00 2001 From: Hicks-99 <94490389+Hicks-99@users.noreply.github.com> Date: Sat, 20 Sep 2025 09:26:19 +0200 Subject: [PATCH 2/2] style(steam): remove emojis from log messages --- server/internal/metadata/steam.ts | 82 +++++++++++++++---------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/server/internal/metadata/steam.ts b/server/internal/metadata/steam.ts index 89601411..5cee2a96 100644 --- a/server/internal/metadata/steam.ts +++ b/server/internal/metadata/steam.ts @@ -234,10 +234,10 @@ export class SteamProvider implements MetadataProvider { { id, publisher, developer, createObject }: _FetchGameMetadataParams, context?: TaskRunContext, ): Promise { - context?.logger.info(`🎮 Starting Steam metadata fetch for game ID: ${id}`); + context?.logger.info(`Starting Steam metadata fetch for game ID: ${id}`); context?.progress(0); - context?.logger.info("📡 Fetching game details from Steam API..."); + context?.logger.info("Fetching game details from Steam API..."); const response = await this._fetchGameDetails([id], { include_assets: true, include_basic_info: true, @@ -249,16 +249,16 @@ export class SteamProvider implements MetadataProvider { }); if (response.length === 0) { - context?.logger.error(`❌ No game found on Steam with ID: ${id}`); + context?.logger.error(`No game found on Steam with ID: ${id}`); throw new Error(`No game found on Steam with id: ${id}`); } const currentGame = response[0] as SteamAppDetailsLarge; - context?.logger.info(`✅ Found game: "${currentGame.name}" on Steam`); + context?.logger.info(`Found game: "${currentGame.name}" on Steam`); context?.progress(10); - context?.logger.info("🖼️ Processing game images and assets..."); + context?.logger.info("Processing game images and assets..."); const { icon, cover, banner, images } = this._processImages( currentGame, createObject, @@ -270,66 +270,66 @@ export class SteamProvider implements MetadataProvider { : new Date(); if (currentGame.release?.steam_release_date) { - context?.logger.info(`📅 Release date: ${released.toLocaleDateString()}`); + context?.logger.info(`Release date: ${released.toLocaleDateString()}`); } else { context?.logger.warn( - "⚠️ No release date found, using current date as fallback", + "No release date found, using current date as fallback", ); } context?.progress(60); context?.logger.info( - `🏷️ Fetching tags from Steam (${currentGame.tagids?.length || 0} tags to process)...`, + `Fetching tags from Steam (${currentGame.tagids?.length || 0} tags to process)...`, ); const tags = await this._getTagNames(currentGame.tagids || []); context?.logger.info( - `✅ Successfully fetched ${tags.length} tags: ${tags.slice(0, 5).join(", ")}${tags.length > 5 ? "..." : ""}`, + `Successfully fetched ${tags.length} tags: ${tags.slice(0, 5).join(", ")}${tags.length > 5 ? "..." : ""}`, ); context?.progress(70); - context?.logger.info("🏢 Processing publishers and developers..."); + context?.logger.info("Processing publishers and developers..."); const publishers = []; const publisherNames = currentGame.basic_info.publishers || []; context?.logger.info( - `📊 Found ${publisherNames.length} publisher(s) to process`, + `Found ${publisherNames.length} publisher(s) to process`, ); for (const pub of publisherNames) { - context?.logger.info(`🔍 Processing publisher: "${pub.name}"`); + context?.logger.info(`Processing publisher: "${pub.name}"`); const comp = await publisher(pub.name); if (!comp) { - context?.logger.warn(`⚠️ Failed to import publisher "${pub.name}"`); + context?.logger.warn(`Failed to import publisher "${pub.name}"`); continue; } publishers.push(comp); - context?.logger.info(`✅ Successfully imported publisher: "${pub.name}"`); + context?.logger.info(`Successfully imported publisher: "${pub.name}"`); } const developers = []; const developerNames = currentGame.basic_info.developers || []; context?.logger.info( - `📊 Found ${developerNames.length} developer(s) to process`, + `Found ${developerNames.length} developer(s) to process`, ); for (const dev of developerNames) { - context?.logger.info(`🔍 Processing developer: "${dev.name}"`); + context?.logger.info(`Processing developer: "${dev.name}"`); const comp = await developer(dev.name); if (!comp) { - context?.logger.warn(`⚠️ Failed to import developer "${dev.name}"`); + context?.logger.warn(`Failed to import developer "${dev.name}"`); continue; } developers.push(comp); - context?.logger.info(`✅ Successfully imported developer: "${dev.name}"`); + context?.logger.info(`Successfully imported developer: "${dev.name}"`); } context?.logger.info( - `✅ Company processing complete: ${publishers.length} publishers, ${developers.length} developers`, + `Company processing complete: ${publishers.length} publishers, ${developers.length} developers`, ); context?.progress(80); - context?.logger.info("📝 Fetching detailed description and reviews..."); + context?.logger.info("Fetching detailed description and reviews..."); const webAppDetails = (await this._getWebAppDetails( id, "metacritic", @@ -342,23 +342,21 @@ export class SteamProvider implements MetadataProvider { let description; if (detailedDescription) { - context?.logger.info("🔄 Converting HTML description to Markdown..."); + context?.logger.info("Converting HTML description to Markdown..."); const converted = this._htmlToMarkdown(detailedDescription, createObject); images.push(...converted.objects); description = converted.markdown; context?.logger.info( - `✅ Description converted, ${converted.objects.length} images embedded`, + `Description converted, ${converted.objects.length} images embedded`, ); } else { - context?.logger.info( - "📄 Using fallback description from basic game info", - ); + context?.logger.info("Using fallback description from basic game info"); description = currentGame.full_description; } context?.progress(90); - context?.logger.info("⭐ Processing review ratings..."); + context?.logger.info("Processing review ratings..."); const reviews = [ { metadataId: id, @@ -375,7 +373,7 @@ export class SteamProvider implements MetadataProvider { const steamRating = currentGame.reviews?.summary_filtered?.percent_positive || 0; context?.logger.info( - `📊 Steam reviews: ${steamReviewCount} reviews, ${steamRating}% positive`, + `Steam reviews: ${steamReviewCount} reviews, ${steamRating}% positive`, ); if (webAppDetails?.metacritic) { @@ -387,16 +385,16 @@ export class SteamProvider implements MetadataProvider { mReviewRating: webAppDetails.metacritic.score / 100, }); context?.logger.info( - `🎯 Metacritic score: ${webAppDetails.metacritic.score}/100`, + `Metacritic score: ${webAppDetails.metacritic.score}/100`, ); } context?.logger.info( - `✅ Review processing complete: ${reviews.length} rating sources found`, + `Review processing complete: ${reviews.length} rating sources found`, ); context?.progress(100); - context?.logger.info("🎉 Steam metadata fetch completed successfully!"); + context?.logger.info("Steam metadata fetch completed successfully!"); return { id: currentGame.appid.toString(), @@ -645,45 +643,45 @@ export class SteamProvider implements MetadataProvider { ): { icon: string; cover: string; banner: string; images: string[] } { const imageURLFormat = game.assets?.asset_url_format; - context?.logger.info("🖼️ Processing game icon..."); + context?.logger.info("Processing game icon..."); let iconRaw; if (game.assets?.community_icon) { - context?.logger.info("✅ Found community icon on Steam"); + context?.logger.info("Found community icon on Steam"); iconRaw = `https://cdn.fastly.steamstatic.com/steamcommunity/public/images/apps/${game.appid}/${game.assets.community_icon}.jpg`; } else { - context?.logger.info("⚠️ No icon found, generating fallback icon"); + context?.logger.info("No icon found, generating fallback icon"); iconRaw = jdenticon.toPng(game.appid, 512); } const icon = createObject(iconRaw); context?.progress(20); - context?.logger.info("🎨 Processing game cover art..."); + context?.logger.info("Processing game cover art..."); let coverRaw; if (game.assets?.library_capsule_2x) { - context?.logger.info("✅ Found high-resolution cover art"); + context?.logger.info("Found high-resolution cover art"); coverRaw = this._getImageUrl( game.assets.library_capsule_2x, imageURLFormat, ); } else if (game.assets?.library_capsule) { - context?.logger.info("✅ Found standard resolution cover art"); + context?.logger.info("Found standard resolution cover art"); coverRaw = this._getImageUrl(game.assets.library_capsule, imageURLFormat); } else { - context?.logger.info("⚠️ No cover art found, generating fallback cover"); + context?.logger.info("No cover art found, generating fallback cover"); coverRaw = jdenticon.toPng(game.appid, 512); } const cover = createObject(coverRaw); context?.progress(30); - context?.logger.info("🏞️ Processing game banner..."); + context?.logger.info("Processing game banner..."); let bannerRaw; if (game.assets?.library_hero) { - context?.logger.info("✅ Found library hero banner"); + context?.logger.info("Found library hero banner"); bannerRaw = this._getImageUrl(game.assets.library_hero, imageURLFormat); } else { - context?.logger.info("⚠️ No banner found, generating fallback banner"); + context?.logger.info("No banner found, generating fallback banner"); bannerRaw = jdenticon.toPng(game.appid, 512); } @@ -692,7 +690,7 @@ export class SteamProvider implements MetadataProvider { const images = [cover, banner]; const screenshotCount = game.screenshots?.all_ages_screenshots?.length || 0; - context?.logger.info(`📸 Processing ${screenshotCount} screenshots...`); + context?.logger.info(`Processing ${screenshotCount} screenshots...`); for (const image of game.screenshots?.all_ages_screenshots || []) { const imageUrl = this._getImageUrl(image.filename); @@ -700,7 +698,7 @@ export class SteamProvider implements MetadataProvider { } context?.logger.info( - `✅ Image processing complete: icon, cover, banner and ${screenshotCount} screenshots`, + `Image processing complete: icon, cover, banner and ${screenshotCount} screenshots`, ); context?.progress(50);