From 802d8fbe47b5a54e24e364d1b640be6509854a36 Mon Sep 17 00:00:00 2001 From: AtlasTheBot Date: Thu, 16 Jul 2020 02:24:36 -0400 Subject: [PATCH] Fix danbooru returning unavailable posts --- CHANGELOG.md | 11 +- package-lock.json | 14 +- package.json | 2 +- src/Constants.ts | 2 +- src/boorus/Booru.ts | 50 ++-- src/index.ts | 17 +- src/structures/InternalSearchParameters.ts | 7 - src/structures/Post.ts | 16 +- src/structures/SearchParameters.ts | 6 + src/structures/SearchResults.ts | 280 ++++++++++----------- test/lolibooru.spec.ts | 36 --- test/util.spec.ts | 6 +- 12 files changed, 205 insertions(+), 242 deletions(-) delete mode 100644 test/lolibooru.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ee02fc7..c7c5565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # booru Changelog -## 2.3.0 [Latest] +## 2.3.2 [Latest] + +- Added `Post#available`, to check if a post isn't deleted/banned +- By default, unavailable posts aren't returned in search results + - You can use `SearchParameters#showUnavailable` to still get them + - `Booru.search('db', ['cat'], { showUnavailable: true })` +- Fix for danbooru occasionally having invalid `fileUrl` or missing IDs + - You can use `Post#available` to check for this + +## 2.3.0 - Fix for illegal invocation errors when using booru on the web - Some of the APIs don't have the required CORS headers however diff --git a/package-lock.json b/package-lock.json index 784c185..d1e7e46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "booru", - "version": "2.3.0", + "version": "2.3.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -6341,9 +6341,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", "dev": true }, "lodash.sortby": { @@ -8256,12 +8256,6 @@ "tsutils": "^2.29.0" }, "dependencies": { - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true - }, "mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", diff --git a/package.json b/package.json index ada3a3f..c4dd2ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "booru", - "version": "2.3.1", + "version": "2.3.2", "description": "Search (and do other things) on a bunch of different boorus!", "author": "AtlasTheBot (https://github.com/AtlasTheBot/)", "license": "MIT", diff --git a/src/Constants.ts b/src/Constants.ts index 23b7ed4..24bff0b 100644 --- a/src/Constants.ts +++ b/src/Constants.ts @@ -4,9 +4,9 @@ */ import { RequestInit } from 'node-fetch' +import siteJson from './sites.json' import Site from './structures/Site' import SiteInfo from './structures/SiteInfo' -import siteJson from './sites.json' export interface SMap { [key: string]: V diff --git a/src/boorus/Booru.ts b/src/boorus/Booru.ts index 9dce2cf..54197ba 100644 --- a/src/boorus/Booru.ts +++ b/src/boorus/Booru.ts @@ -73,20 +73,22 @@ export class Booru { /** * Search for images on this booru * @param {String|String[]} tags The tag(s) to search for - * @param {Object} searchArgs The arguments for the search - * @param {Number} [searchArgs.limit=1] The number of images to return - * @param {Boolean} [searchArgs.random=false] If it should randomly grab results - * @param {Number} [searchArgs.page=0] The page to search + * @param {SearchParameters} searchArgs The arguments for the search * @return {Promise} The results as an array of Posts */ - public async search(tags: string | string[], {limit = 1, random = false, page = 0} - : SearchParameters = {}): Promise { + public async search(tags: string | string[], { + limit = 1, random = false, + page = 0, showUnavailable = false } : SearchParameters = {}): Promise { const fakeLimit: number = random && !this.site.random ? 100 : 0 try { - const searchResult = await this.doSearchRequest(tags, {limit, random, page}) - return this.parseSearchResult(searchResult, {fakeLimit, tags, limit, random, page}) + const searchResult = await this.doSearchRequest(tags, + { limit, random, page, showUnavailable } + ) + return this.parseSearchResult(searchResult, + { fakeLimit, tags, limit, random, page, showUnavailable } + ) } catch (err) { throw new BooruError(err) } @@ -111,16 +113,12 @@ export class Booru { * * @protected * @param {String[]|String} tags The tags to search with - * @param {Object} searchArgs The arguments for the search - * @param {Number} [searchArgs.limit=1] The number of images to return - * @param {Boolean} [searchArgs.random=false] If it should randomly grab results - * @param {Number} [searchArgs.page=0] The page number to search - * @param {String?} [searchArgs.uri=null] If the uri should be overwritten + * @param {InternalSearchParameters} searchArgs The arguments for the search * @return {Promise} */ - protected async doSearchRequest(tags: string[] | string, - {uri = null, limit = 1, random = false, page = 0} - : InternalSearchParameters = {}): Promise { + protected async doSearchRequest(tags: string[] | string, { + uri = null, limit = 1, random = false, page = 0, + } : InternalSearchParameters = {}): Promise { if (!Array.isArray(tags)) tags = [tags] // Used for random on sites without order:random @@ -168,16 +166,12 @@ export class Booru { * * @protected * @param {Object} result The response of the booru - * @param {Object} searchArgs The arguments used for the search - * @param {Number?} [searchArgs.fakeLimit] If the `order:random` should be faked - * @param {String[]|String} [searchArgs.tags] The tags used on the search - * @param {Number} [searchArgs.limit] The number of images to return - * @param {Boolean} [searchArgs.random] If it should randomly grab results - * @param {Number} [searchArgs.page] The page number searched + * @param {InternalSearchParameters} searchArgs The arguments used for the search * @return {SearchResults} The results of this search */ - protected parseSearchResult(result: any, {fakeLimit, tags, limit, random, page} - : InternalSearchParameters) { + protected parseSearchResult(result: any, { + fakeLimit, tags, limit, random, page, showUnavailable, + } : InternalSearchParameters) { if (result.success === false) { throw new BooruError(result.message || result.reason) @@ -202,8 +196,8 @@ export class Booru { } const results = r || result - const posts = results.slice(0, limit).map((v: any) => new Post(v, this)) - const options = {limit, random, page} + let posts: Post[] = results.slice(0, limit).map((v: any) => new Post(v, this)) + const options = { limit, random, page, showUnavailable } if (tags === undefined) { tags = [] @@ -213,6 +207,10 @@ export class Booru { tags = [tags] } + if (!showUnavailable) { + posts = posts.filter(p => p.available) + } + return new SearchResults(posts, tags, options, this) } } diff --git a/src/index.ts b/src/index.ts index e103aea..4672e0d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,16 +3,16 @@ * @module Index */ -import { BooruError, SMap, sites } from './Constants' +import { BooruError, sites, SMap } from './Constants' +import { deprecate } from 'util' import Booru from './boorus/Booru' import Derpibooru from './boorus/Derpibooru' +import XmlBooru from './boorus/XmlBooru' import Post from './structures/Post' import SearchParameters from './structures/SearchParameters' import SearchResults from './structures/SearchResults' import Site from './structures/Site' -import XmlBooru from './boorus/XmlBooru' -import { deprecate } from 'util' import { resolveSite } from './Utils' const BooruTypes: any = { @@ -61,10 +61,7 @@ export default booruForSite * Searches a site for images with tags and returns the results * @param {String} site The site to search * @param {String[]|String} [tags=[]] Tags to search with - * @param {Object} [searchOptions={}] The options for searching - * @param {Number|String} [searchOptions.limit=1] The limit of images to return - * @param {Boolean} [searchOptions.random=false] If it should grab randomly sorted results - * @param {Object?} [searchOptions.credentials=null] Credentials to use to search the booru, + * @param {SearchParameters} [searchOptions={}] The options for searching * if provided (Unused) * @return {Promise} A promise with the images as an array of objects * @@ -75,9 +72,9 @@ export default booruForSite * Booru.search('e926', ['glaceon', 'cute']) * ``` */ -export function search(site: string, tags: string[] | string = [], - {limit = 1, random = false, page = 0, credentials = null} - : SearchParameters = {}): Promise { +export function search(site: string, tags: string[] | string = [], { + limit = 1, random = false, page = 0, credentials = null, showUnavailable = false, +} : SearchParameters = {}): Promise { const rSite: string | null = resolveSite(site) diff --git a/src/structures/InternalSearchParameters.ts b/src/structures/InternalSearchParameters.ts index 69aa5a2..20c2cf0 100644 --- a/src/structures/InternalSearchParameters.ts +++ b/src/structures/InternalSearchParameters.ts @@ -9,13 +9,6 @@ import SearchParameters from './SearchParameters' * Interface for {@link Booru}'s **private internal** search params pls no use */ -/** - * @param {Number?} [searchArgs.fakeLimit] If the `order:random` should be faked - * @param {String[]|String} [searchArgs.tags] The tags used on the search - * @param {Number} [searchArgs.limit] The number of images to return - * @param {Boolean} [searchArgs.random] If it should randomly grab results - * @param {Number} [searchArgs.page] The page number searched - */ export default interface InternalSearchParameters extends SearchParameters { /** The uri to override with, if provided */ uri?: string | null diff --git a/src/structures/Post.ts b/src/structures/Post.ts index 0c6ec12..3f914ea 100644 --- a/src/structures/Post.ts +++ b/src/structures/Post.ts @@ -49,11 +49,6 @@ function parseImageUrl(url: string, data: any, booru: Booru): string | null { url = `https:${url}` } - // Lolibooru likes to shove all the tags into its urls, despite the fact you don't need the tags - if (url.match(/https?:\/\/lolibooru.moe/)) { - url = data.sample_url.replace(/(.*booru \d+ ).*(\..*)/, '$1sample$2') - } - return encodeURI(url) } @@ -122,6 +117,8 @@ export default class Post { public previewWidth: number | null /** The id of this post */ public id: string + /** If this post is available (ie. not deleted, not banned, has file url) */ + public available: boolean /** The tags of this post */ public tags: string[] /** The score of this post */ @@ -150,12 +147,17 @@ export default class Post { this.data = data this.booru = booru + // Again, thanks danbooru + const deletedOrBanned = data.is_deleted || data.is_banned + this.fileUrl = parseImageUrl( - data.file_url || data.image || data.source + data.file_url || data.image || (deletedOrBanned ? data.source : undefined) || (data.file && data.file.url) || (data.representations && data.representations.full), data, booru) + this.available = !deletedOrBanned && this.fileUrl !== null + this.height = parseInt(data.height || data.image_height || (data.file && data.file.height), 10) this.width = parseInt(data.width || data.image_width || (data.file && data.file.width), 10) @@ -175,7 +177,7 @@ export default class Post { this.previewHeight = parseInt(data.preview_height || (data.preview && data.preview.height), 10) this.previewWidth = parseInt(data.preview_width || (data.preview && data.preview.width), 10) - this.id = data.id.toString() + this.id = data.id ? data.id.toString() : 'No ID available' this.tags = getTags(data) // Too long for conditional diff --git a/src/structures/SearchParameters.ts b/src/structures/SearchParameters.ts index 4449865..69f73f3 100644 --- a/src/structures/SearchParameters.ts +++ b/src/structures/SearchParameters.ts @@ -7,8 +7,14 @@ * Just an interface for {@link Booru}'s search params :) */ export default interface SearchParameters { + /** The limit on *max* posts to show, you might get less posts than this */ limit?: number + /** Should posts be in random order, implementation differs per booru */ random?: boolean + /** Which page of results to fetch */ page?: number + /** The credentials to use to auth with the booru */ credentials?: any + /** Return unavailable posts (ie. banned/deleted posts) */ + showUnavailable?: boolean } diff --git a/src/structures/SearchResults.ts b/src/structures/SearchResults.ts index e51ff3a..d1458a4 100644 --- a/src/structures/SearchResults.ts +++ b/src/structures/SearchResults.ts @@ -1,140 +1,140 @@ -/** - * @packageDocumentation - * @module Structures - */ - -import Booru from '../boorus/Booru' -import Post from '../structures/Post' -import * as Utils from '../Utils' -import SearchParameters from './SearchParameters' - -/** - * Represents a page of search results, works like an array of {@link Post} - *

Usable like an array and allows to easily get the next page - * - * @example - * ``` - * const Booru = require('booru') - * // Safebooru - * const sb = new Booru('sb') - * - * const imgs = await sb.search('cat') - * - * // Log the images from the first page, then from the second - * imgs.forEach(i => console.log(i.postView)) - * const imgs2 = await imgs.nextPage() - * imgs2.forEach(i => console.log(i.postView)) - * ``` - */ -class SearchResults extends Array { - /** The booru used for this search */ - public booru: Booru - /** The page of this search */ - public page: number - /** The tags used for this search */ - public readonly tags: string[] - /** The options used for this search */ - public readonly options: SearchParameters - /** The posts from this search result */ - public readonly posts: Post[] - - /** @private */ - constructor(posts: Post[], tags: string[], options: SearchParameters, booru: Booru) { - super(posts.length) - - for (let i: number = 0; i < posts.length; i++) { - this[i] = posts[i] - } - - this.posts = posts - this.tags = tags - this.options = options - this.booru = booru - this.page = options ? options.page || 0 : 0 - } - - /** - * Get the first post in this result set - * @return {Post} - */ - get first(): Post { - return this[0] - } - - /** - * Get the last post in this result set - * @return {Post} - */ - get last(): Post { - return this[this.length - 1] - } - - /** - * Get the next page - *

Works like sb.search('cat', {page: 1}); sb.search('cat', {page: 2}) - * @return {Promise} - */ - public nextPage(): Promise { - const opts: SearchParameters = this.options - opts.page = this.page + 1 - - return this.booru.search(this.tags, opts) - } - - /** - * Create a new SearchResults with just images with the matching tags - * - * @param {String[]|String} tags The tags (or tag) to search for - * @param {Object} options The extra options for the search - * @param {Boolean} [options.invert=false] If the results should be inverted and - * return images *not* tagged - * @return {SearchResults} - */ - public tagged(tags: string[] | string, {invert = false} = {}): SearchResults { - if (!Array.isArray(tags)) { - tags = [tags] - } - - const posts: Post[] = [] - - for (const p of this) { - const m: number = Utils.compareArrays(tags, p.tags).length - if ((!invert && m > 0) || (invert && m === 0)) { - posts.push(p) - } - } - - return new SearchResults(posts, this.tags, this.options, this.booru) - } - - /** - * Returns a SearchResults with images *not* tagged with any of the specified tags (or tag) - * @param {String[]|String} tags The tags (or tag) to blacklist - * @return {SearchResults} The results without any images with the specified tags - */ - public blacklist(tags: string[] | string): SearchResults { - return this.tagged(tags, {invert: true}) - } -} - -// Workaround for the odd behavior as it extends Array -// Calling an array function on the result will cause it to call the constructor for SearchResults -// With the incorrect params (ie. new SearchResults(0)) thinking it's an array -const prototypeKeys: string[] = Reflect - .ownKeys(Array.prototype) - .filter(k => typeof k === 'string' && k !== 'constructor') as unknown as string[] - -// Are you ready for hell of a workaround? -for (const p of prototypeKeys) { - if (typeof Array.prototype[p as any] === 'function') { - const proxy = function(this: SearchResults, ...args: any[]) { - // tslint:disable-next-line: ban-types - return (this.posts[p as any] as unknown as Function)(...args) - } - - // See https://github.com/AtlasTheBot/booru/issues/38 - Object.defineProperty(SearchResults.prototype, p, { value: proxy }) - } -} - -export default SearchResults +/** + * @packageDocumentation + * @module Structures + */ + +import Booru from '../boorus/Booru' +import Post from '../structures/Post' +import * as Utils from '../Utils' +import SearchParameters from './SearchParameters' + +/** + * Represents a page of search results, works like an array of {@link Post} + *

Usable like an array and allows to easily get the next page + * + * @example + * ``` + * const Booru = require('booru') + * // Safebooru + * const sb = new Booru('sb') + * + * const imgs = await sb.search('cat') + * + * // Log the images from the first page, then from the second + * imgs.forEach(i => console.log(i.postView)) + * const imgs2 = await imgs.nextPage() + * imgs2.forEach(i => console.log(i.postView)) + * ``` + */ +class SearchResults extends Array { + /** The booru used for this search */ + public booru: Booru + /** The page of this search */ + public page: number + /** The tags used for this search */ + public readonly tags: string[] + /** The options used for this search */ + public readonly options: SearchParameters + /** The posts from this search result */ + public readonly posts: Post[] + + /** @private */ + constructor(posts: Post[], tags: string[], options: SearchParameters, booru: Booru) { + super(posts.length) + + for (let i: number = 0; i < posts.length; i++) { + this[i] = posts[i] + } + + this.posts = posts + this.tags = tags + this.options = options + this.booru = booru + this.page = options ? options.page || 0 : 0 + } + + /** + * Get the first post in this result set + * @return {Post} + */ + get first(): Post { + return this[0] + } + + /** + * Get the last post in this result set + * @return {Post} + */ + get last(): Post { + return this[this.length - 1] + } + + /** + * Get the next page + *

Works like sb.search('cat', {page: 1}); sb.search('cat', {page: 2}) + * @return {Promise} + */ + public nextPage(): Promise { + const opts: SearchParameters = this.options + opts.page = this.page + 1 + + return this.booru.search(this.tags, opts) + } + + /** + * Create a new SearchResults with just images with the matching tags + * + * @param {String[]|String} tags The tags (or tag) to search for + * @param {Object} options The extra options for the search + * @param {Boolean} [options.invert=false] If the results should be inverted and + * return images *not* tagged + * @return {SearchResults} + */ + public tagged(tags: string[] | string, {invert = false} = {}): SearchResults { + if (!Array.isArray(tags)) { + tags = [tags] + } + + const posts: Post[] = [] + + for (const p of this) { + const m: number = Utils.compareArrays(tags, p.tags).length + if ((!invert && m > 0) || (invert && m === 0)) { + posts.push(p) + } + } + + return new SearchResults(posts, this.tags, this.options, this.booru) + } + + /** + * Returns a SearchResults with images *not* tagged with any of the specified tags (or tag) + * @param {String[]|String} tags The tags (or tag) to blacklist + * @return {SearchResults} The results without any images with the specified tags + */ + public blacklist(tags: string[] | string): SearchResults { + return this.tagged(tags, {invert: true}) + } +} + +// Workaround for the odd behavior as it extends Array +// Calling an array function on the result will cause it to call the constructor for SearchResults +// With the incorrect params (ie. new SearchResults(0)) thinking it's an array +const prototypeKeys: string[] = Reflect + .ownKeys(Array.prototype) + .filter(k => typeof k === 'string' && k !== 'constructor') as unknown as string[] + +// Are you ready for hell of a workaround? +for (const p of prototypeKeys) { + if (typeof Array.prototype[p as any] === 'function') { + const proxy = function(this: SearchResults, ...args: any[]) { + // tslint:disable-next-line: ban-types + return (this.posts[p as any] as unknown as Function)(...args) + } + + // See https://github.com/AtlasTheBot/booru/issues/38 + Object.defineProperty(SearchResults.prototype, p, { value: proxy }) + } +} + +export default SearchResults diff --git a/test/lolibooru.spec.ts b/test/lolibooru.spec.ts deleted file mode 100644 index 327c97d..0000000 --- a/test/lolibooru.spec.ts +++ /dev/null @@ -1,36 +0,0 @@ -import Booru, { BooruClass, search, sites } from '../src/index'; -import Post from '../src/structures/Post'; -import SearchResults from '../src/structures/SearchResults'; - -let tag1: string; -let site: string; - -beforeEach(() => { - site = 'lb'; - tag1 = 'glaceon'; -}); - -describe('Using instantiation method', () => { - let danbooru: BooruClass; - beforeEach(() => { - danbooru = Booru(site); - }); - - it('should return an image', async () => { - const searchResult: SearchResults = await danbooru.search([tag1]); - const image: Post = searchResult[0]; - expect(searchResult.booru.domain).toBe('lolibooru.moe'); - expect(searchResult.booru.site).toMatchObject(sites[searchResult.booru.domain]); - expect(typeof image.fileUrl).toBe('string'); - }); -}); - -describe('Using fancy pants method', () => { - it('should return an image', async () => { - const searchResult = await search(site, [tag1]); - const image: Post = searchResult[0]; - expect(searchResult.booru.domain).toBe('lolibooru.moe'); - expect(searchResult.booru.site).toMatchObject(sites[searchResult.booru.domain]); - expect(typeof image.fileUrl).toBe('string'); - }); -}); \ No newline at end of file diff --git a/test/util.spec.ts b/test/util.spec.ts index c71043d..3177cfe 100644 --- a/test/util.spec.ts +++ b/test/util.spec.ts @@ -40,8 +40,8 @@ describe('check BooruClass', () => { }); describe('check sites', () => { - it('should support 17 sites', () => { + it('should support 16 sites', () => { const map = sites; - expect(Object.keys(map)).toHaveLength(17); + expect(Object.keys(map)).toHaveLength(16); }); -}); \ No newline at end of file +});