From 0ef54b044c7b10ff3a6abc206cc45f10b5f4665a Mon Sep 17 00:00:00 2001 From: djdembeck Date: Wed, 10 Aug 2022 17:36:03 -0500 Subject: [PATCH] feat(helper): :sparkles: use Audible API for genres fallback to html genres when no API genres available Lots more coverage needs to be added to make sure fallbacks work properly --- src/helpers/books/audible/ApiHelper.ts | 51 ++++++++++++++++++- src/helpers/books/audible/StitchHelper.ts | 24 +++++++-- tests/audible/books/api.test.ts | 4 +- tests/helpers/books/audible/ApiHelper.test.ts | 2 +- .../books/audible/StitchHelper.test.ts | 15 +----- 5 files changed, 73 insertions(+), 23 deletions(-) diff --git a/src/helpers/books/audible/ApiHelper.ts b/src/helpers/books/audible/ApiHelper.ts index f381605f..b406a5fe 100644 --- a/src/helpers/books/audible/ApiHelper.ts +++ b/src/helpers/books/audible/ApiHelper.ts @@ -1,10 +1,11 @@ import { htmlToText } from 'html-to-text' -import { AudibleProduct, AudibleSeries } from '#config/typing/audible' -import { ApiBook, Series } from '#config/typing/books' +import { AudibleProduct, AudibleSeries, Category } from '#config/typing/audible' +import { ApiBook, ApiGenre, Series } from '#config/typing/books' import { AuthorOnBook, NarratorOnBook } from '#config/typing/people' import fetch from '#helpers/fetchPlus' import SharedHelper from '#helpers/shared' +import { parentCategories } from '#static/constants' class ApiHelper { asin: string @@ -16,6 +17,7 @@ class ApiHelper { const baseDomain = 'https://api.audible.com' const baseUrl = '1.0/catalog/products' const paramArr = [ + 'category_ladders', 'contributors', 'product_desc', 'product_extended_attrs', @@ -52,6 +54,46 @@ class ApiHelper { }) } + isParentCategory(category: Category): boolean { + return parentCategories.some((parentCategory) => { + return parentCategory.id === category.id && parentCategory.name === category.name + }) + } + + categoryToApiGenre(category: Category, type: string): ApiGenre { + return { + asin: category.id, + name: category.name, + type: type + } + } + + getGenres(categories: Category[]): ApiGenre[] { + // Genres ARE parent categories + const filtered = categories.filter(this.isParentCategory) + // Transform categories to ApiGenres + return filtered.map((category) => { + return this.categoryToApiGenre(category, 'genre') + }) + } + + getTags(categories: Category[]): ApiGenre[] { + // Tags are NOT parent categories + const filtered = categories.filter((e) => !this.isParentCategory(e)) + // Transform categories to ApiGenres + return filtered.map((category) => { + return this.categoryToApiGenre(category, 'tag') + }) + } + + getCategories(): Category[] { + if (!this.inputJson) throw new Error(`No input data`) + // Flatten category ladders to a single array of categories + const categories = this.inputJson.category_ladders?.map((category) => category.ladder).flat() + // Remove duplicates from categories array + return [...new Map(categories.map((item) => [item.name, item])).values()] + } + getHighResImage() { if (!this.inputJson) throw new Error(`No input data`) return this.inputJson.product_images?.[1024] @@ -119,6 +161,8 @@ class ApiHelper { getFinalData(): ApiBook { if (!this.inputJson) throw new Error(`No input data`) if (!this.inputJson.title) throw new Error(`No title`) + // Get flattened categories + const categories = this.getCategories() // Find secondary series if available const series1 = this.getSeriesPrimary(this.inputJson.series) const series2 = this.getSeriesSecondary(this.inputJson.series) @@ -136,6 +180,9 @@ class ApiHelper { wordwrap: false }).trim(), formatType: this.inputJson.format_type, + ...(categories && { + genres: [...this.getGenres(categories), ...this.getTags(categories)] + }), image: this.getHighResImage(), language: this.inputJson.language, ...(this.inputJson.narrators && { diff --git a/src/helpers/books/audible/StitchHelper.ts b/src/helpers/books/audible/StitchHelper.ts index 0dd5392d..a10e35dc 100644 --- a/src/helpers/books/audible/StitchHelper.ts +++ b/src/helpers/books/audible/StitchHelper.ts @@ -31,6 +31,10 @@ class StitchHelper { // Run fetch tasks in parallel try { this.apiResponse = await apiResponse + // Skip scraping if API response has category ladders + if (this.apiResponse?.product.category_ladders.length) { + return + } this.scraperResponse = await scraperResponse } catch (err) { throw new Error(`Error occured while fetching data from API or scraper: ${err}`) @@ -42,13 +46,20 @@ class StitchHelper { */ async parseResponses() { const apiParsed = this.apiHelper.parseResponse(this.apiResponse) - const scraperParsed = this.scrapeHelper.parseResponse(this.scraperResponse) + // Skip scraper parsing if API response has category ladders + let scraperParsed: Promise | undefined = undefined + if (this.scraperResponse) { + console.debug( + `API response has no category ladders, parsing scraper response for: ${this.asin}` + ) + scraperParsed = this.scrapeHelper.parseResponse(this.scraperResponse) + } // Run parse tasks in parallel try { this.apiParsed = await apiParsed // Also create the partial json for genre use - this.scraperParsed = await scraperParsed + this.scraperParsed = scraperParsed ? await scraperParsed : undefined } catch (err) { throw new Error(`Error occured while parsing data from API or scraper: ${err}`) } @@ -73,9 +84,14 @@ class StitchHelper { await this.fetchSources() await this.parseResponses() + // If parsed API response has genres, return it + if (this.apiParsed?.genres?.length) { + return this.apiParsed as Book + } + + // If no genres in API response, return scraper parsed response const stitchedGenres = await this.includeGenres() - const bookJson: Book = stitchedGenres - return bookJson + return stitchedGenres } } diff --git a/tests/audible/books/api.test.ts b/tests/audible/books/api.test.ts index 299fac32..52e6d00c 100644 --- a/tests/audible/books/api.test.ts +++ b/tests/audible/books/api.test.ts @@ -62,7 +62,7 @@ describe('Audible API', () => { const description = "James Patterson's Detective Billy Harney is back, this time investigating murders in a notorious Chicago drug ring, which will lead him, his sister, and his new partner through a dangerous web of corrupt politicians, vengeful billionaires, and violent dark web conspiracies...." const image = 'https://m.media-amazon.com/images/I/91H9ynKGNwL.jpg' - minimalParsed = setupMinimalParsed(B08C6YJ1LS.product, description, image) + minimalParsed = setupMinimalParsed(B08C6YJ1LS.product, description, image, parsed.genres) }) it('returned the correct data', () => { @@ -82,7 +82,7 @@ describe('Audible API', () => { const description = 'Harry Potter has never even heard of Hogwarts when the letters start dropping on the doormat at number four, Privet Drive. Addressed in green ink on yellowish parchment with a purple seal, they are swiftly confiscated by his grisly aunt and uncle....' const image = 'https://m.media-amazon.com/images/I/91eopoUCjLL.jpg' - minimalParsed = setupMinimalParsed(B017V4IM1G.product, description, image) + minimalParsed = setupMinimalParsed(B017V4IM1G.product, description, image, parsed.genres) }) it('returned the correct data', () => { diff --git a/tests/helpers/books/audible/ApiHelper.test.ts b/tests/helpers/books/audible/ApiHelper.test.ts index 5657dac1..8ce6d169 100644 --- a/tests/helpers/books/audible/ApiHelper.test.ts +++ b/tests/helpers/books/audible/ApiHelper.test.ts @@ -16,7 +16,7 @@ describe('ApiHelper should', () => { test('setup constructor correctly', () => { expect(helper.asin).toBe(asin) expect(helper.reqUrl).toBe( - `https://api.audible.com/1.0/catalog/products/${asin}/?response_groups=contributors,product_desc,product_extended_attrs,product_attrs,media,rating,series&image_sizes=500,1024` + `https://api.audible.com/1.0/catalog/products/${asin}/?response_groups=category_ladders,contributors,product_desc,product_extended_attrs,product_attrs,media,rating,series&image_sizes=500,1024` ) }) diff --git a/tests/helpers/books/audible/StitchHelper.test.ts b/tests/helpers/books/audible/StitchHelper.test.ts index 8af70560..a4cbcd48 100644 --- a/tests/helpers/books/audible/StitchHelper.test.ts +++ b/tests/helpers/books/audible/StitchHelper.test.ts @@ -1,11 +1,7 @@ -import * as cheerio from 'cheerio' - import ApiHelper from '#helpers/books/audible/ApiHelper' -import ScrapeHelper from '#helpers/books/audible/ScrapeHelper' import StitchHelper from '#helpers/books/audible/StitchHelper' import { apiResponse, - genresObject, htmlResponse, parsedBook, parsedBookWithGenres @@ -33,36 +29,27 @@ describe('StitchHelper should', () => { test('setup constructor correctly', () => { expect(helper.asin).toBe(asin) expect(helper.apiHelper).toBeInstanceOf(ApiHelper) - expect(helper.scrapeHelper).toBeInstanceOf(ScrapeHelper) }) test('fetch sources', async () => { await helper.fetchSources() expect(helper.apiResponse).toEqual(apiResponse) - expect(helper.scraperResponse.html()).toEqual(cheerio.load(htmlResponse).html()) }) test('parse responses', async () => { await helper.fetchSources() await helper.parseResponses() expect(helper.apiParsed).toEqual(parsedBook) - expect(helper.scraperParsed).toEqual(genresObject) }) - test('include genres if genres exist', async () => { - await helper.fetchSources() - await helper.parseResponses() - await expect(helper.includeGenres()).resolves.toEqual(parsedBookWithGenres) - }) + test.todo('parse html genres') test('process book', async () => { const proccessed = await helper.process() expect(proccessed).toEqual(parsedBookWithGenres) expect(helper.apiResponse).toEqual(apiResponse) - expect(helper.scraperResponse.html()).toEqual(cheerio.load(htmlResponse).html()) expect(helper.apiParsed).toEqual(parsedBook) - expect(helper.scraperParsed).toEqual(genresObject) }) })