Fix danbooru returning unavailable posts

CHANGELOG.md

@@ -1,6 +1,15 @@
 # booru Changelog
 
-## 2.3.0 [Latest]
+## 2.3.2 [Latest]
+
+- Added `Post#available`, to check if a post isn't deleted/banned
+- By default, unavailable posts aren't returned in search results
+  - You can use `SearchParameters#showUnavailable` to still get them
+  - `Booru.search('db', ['cat'], { showUnavailable: true })`
+- Fix for danbooru occasionally having invalid `fileUrl` or missing IDs
+  - You can use `Post#available` to check for this
+
+## 2.3.0
 
 - Fix for illegal invocation errors when using booru on the web
   - Some of the APIs don't have the required CORS headers however

package.json

@@ -1,6 +1,6 @@
 {
   "name": "booru",
-  "version": "2.3.1",
+  "version": "2.3.2",
   "description": "Search (and do other things) on a bunch of different boorus!",
   "author": "AtlasTheBot (https://github.com/AtlasTheBot/)",
   "license": "MIT",

src/Constants.ts

@@ -4,9 +4,9 @@
  */
 
 import { RequestInit } from 'node-fetch'
+import siteJson from './sites.json'
 import Site from './structures/Site'
 import SiteInfo from './structures/SiteInfo'
-import siteJson from './sites.json'
 
 export interface SMap<V> {
   [key: string]: V

src/boorus/Booru.ts

@@ -73,20 +73,22 @@ export class Booru {
   /**
    * Search for images on this booru
    * @param {String|String[]} tags The tag(s) to search for
-   * @param {Object} searchArgs The arguments for the search
-   * @param {Number} [searchArgs.limit=1] The number of images to return
-   * @param {Boolean} [searchArgs.random=false] If it should randomly grab results
-   * @param {Number} [searchArgs.page=0] The page to search
+   * @param {SearchParameters} searchArgs The arguments for the search
    * @return {Promise<SearchResults>} The results as an array of Posts
    */
-  public async search(tags: string | string[], {limit = 1, random = false, page = 0}
-                       : SearchParameters = {}): Promise<SearchResults> {
+  public async search(tags: string | string[], {
+    limit = 1, random = false,
+    page = 0, showUnavailable = false } : SearchParameters = {}): Promise<SearchResults> {
 
     const fakeLimit: number = random && !this.site.random ? 100 : 0
 
     try {
-      const searchResult = await this.doSearchRequest(tags, {limit, random, page})
-      return this.parseSearchResult(searchResult, {fakeLimit, tags, limit, random, page})
+      const searchResult = await this.doSearchRequest(tags,
+        { limit, random, page, showUnavailable }
+      )
+      return this.parseSearchResult(searchResult,
+        { fakeLimit, tags, limit, random, page, showUnavailable }
+      )
     } catch (err) {
       throw new BooruError(err)
     }
@@ -111,16 +113,12 @@ export class Booru {
    *
    * @protected
    * @param {String[]|String} tags The tags to search with
-   * @param {Object} searchArgs The arguments for the search
-   * @param {Number} [searchArgs.limit=1] The number of images to return
-   * @param {Boolean} [searchArgs.random=false] If it should randomly grab results
-   * @param {Number} [searchArgs.page=0] The page number to search
-   * @param {String?} [searchArgs.uri=null] If the uri should be overwritten
+   * @param {InternalSearchParameters} searchArgs The arguments for the search
    * @return {Promise<Object>}
    */
-  protected async doSearchRequest(tags: string[] | string,
-                                  {uri = null, limit = 1, random = false, page = 0}
-                                   : InternalSearchParameters = {}): Promise<any> {
+  protected async doSearchRequest(tags: string[] | string, {
+    uri = null, limit = 1, random = false, page = 0,
+  } : InternalSearchParameters = {}): Promise<any> {
     if (!Array.isArray(tags)) tags = [tags]
 
     // Used for random on sites without order:random
@@ -168,16 +166,12 @@ export class Booru {
    *
    * @protected
    * @param {Object} result The response of the booru
-   * @param {Object} searchArgs The arguments used for the search
-   * @param {Number?} [searchArgs.fakeLimit] If the `order:random` should be faked
-   * @param {String[]|String} [searchArgs.tags] The tags used on the search
-   * @param {Number} [searchArgs.limit] The number of images to return
-   * @param {Boolean} [searchArgs.random] If it should randomly grab results
-   * @param {Number} [searchArgs.page] The page number searched
+   * @param {InternalSearchParameters} searchArgs The arguments used for the search
    * @return {SearchResults} The results of this search
    */
-  protected parseSearchResult(result: any, {fakeLimit, tags, limit, random, page}
-                               : InternalSearchParameters) {
+  protected parseSearchResult(result: any, {
+    fakeLimit, tags, limit, random, page, showUnavailable,
+  } : InternalSearchParameters) {
 
     if (result.success === false) {
       throw new BooruError(result.message || result.reason)
@@ -202,8 +196,8 @@ export class Booru {
     }
 
     const results = r || result
-    const posts = results.slice(0, limit).map((v: any) => new Post(v, this))
-    const options = {limit, random, page}
+    let posts: Post[] = results.slice(0, limit).map((v: any) => new Post(v, this))
+    const options = { limit, random, page, showUnavailable }
 
     if (tags === undefined) {
       tags = []
@@ -213,6 +207,10 @@ export class Booru {
       tags = [tags]
     }
 
+    if (!showUnavailable) {
+      posts = posts.filter(p => p.available)
+    }
+
     return new SearchResults(posts, tags, options, this)
   }
 }

src/index.ts

@@ -3,16 +3,16 @@
  * @module Index
  */
 
-import { BooruError, SMap, sites } from './Constants'
+import { BooruError, sites, SMap } from './Constants'
 
+import { deprecate } from 'util'
 import Booru from './boorus/Booru'
 import Derpibooru from './boorus/Derpibooru'
+import XmlBooru from './boorus/XmlBooru'
 import Post from './structures/Post'
 import SearchParameters from './structures/SearchParameters'
 import SearchResults from './structures/SearchResults'
 import Site from './structures/Site'
-import XmlBooru from './boorus/XmlBooru'
-import { deprecate } from 'util'
 import { resolveSite } from './Utils'
 
 const BooruTypes: any = {
@@ -61,10 +61,7 @@ export default booruForSite
  * Searches a site for images with tags and returns the results
  * @param {String} site The site to search
  * @param {String[]|String} [tags=[]] Tags to search with
- * @param {Object} [searchOptions={}] The options for searching
- * @param {Number|String} [searchOptions.limit=1] The limit of images to return
- * @param {Boolean} [searchOptions.random=false] If it should grab randomly sorted results
- * @param {Object?} [searchOptions.credentials=null] Credentials to use to search the booru,
+ * @param {SearchParameters} [searchOptions={}] The options for searching
  *  if provided (Unused)
  * @return {Promise<SearchResults>} A promise with the images as an array of objects
  *
@@ -75,9 +72,9 @@ export default booruForSite
  * Booru.search('e926', ['glaceon', 'cute'])
  * ```
  */
-export function search(site: string, tags: string[] | string = [],
-                       {limit = 1, random = false, page = 0, credentials = null}
-                        : SearchParameters = {}): Promise<SearchResults> {
+export function search(site: string, tags: string[] | string = [], {
+  limit = 1, random = false, page = 0, credentials = null, showUnavailable = false,
+} : SearchParameters = {}): Promise<SearchResults> {
 
   const rSite: string | null = resolveSite(site)
 

src/structures/InternalSearchParameters.ts

@@ -9,13 +9,6 @@ import SearchParameters from './SearchParameters'
  * Interface for {@link Booru}'s **private internal** search params pls no use
  */
 
-/**
- * @param {Number?} [searchArgs.fakeLimit] If the `order:random` should be faked
- * @param {String[]|String} [searchArgs.tags] The tags used on the search
- * @param {Number} [searchArgs.limit] The number of images to return
- * @param {Boolean} [searchArgs.random] If it should randomly grab results
- * @param {Number} [searchArgs.page] The page number searched
- */
 export default interface InternalSearchParameters extends SearchParameters {
   /** The uri to override with, if provided */
   uri?: string | null

src/structures/Post.ts

@@ -49,11 +49,6 @@ function parseImageUrl(url: string, data: any, booru: Booru): string | null {
     url = `https:${url}`
   }
 
-  // Lolibooru likes to shove all the tags into its urls, despite the fact you don't need the tags
-  if (url.match(/https?:\/\/lolibooru.moe/)) {
-    url = data.sample_url.replace(/(.*booru \d+ ).*(\..*)/, '$1sample$2')
-  }
-
   return encodeURI(url)
 }
 
@@ -122,6 +117,8 @@ export default class Post {
   public previewWidth: number | null
   /** The id of this post */
   public id: string
+  /** If this post is available (ie. not deleted, not banned, has file url) */
+  public available: boolean
   /** The tags of this post */
   public tags: string[]
   /** The score of this post */
@@ -150,12 +147,17 @@ export default class Post {
     this.data = data
     this.booru = booru
 
+    // Again, thanks danbooru
+    const deletedOrBanned = data.is_deleted || data.is_banned
+
     this.fileUrl = parseImageUrl(
-      data.file_url || data.image || data.source
+      data.file_url || data.image || (deletedOrBanned ? data.source : undefined)
       || (data.file && data.file.url)
       || (data.representations && data.representations.full),
       data, booru)
 
+    this.available = !deletedOrBanned && this.fileUrl !== null
+
     this.height = parseInt(data.height || data.image_height || (data.file && data.file.height), 10)
     this.width = parseInt(data.width || data.image_width || (data.file && data.file.width), 10)
 
@@ -175,7 +177,7 @@ export default class Post {
     this.previewHeight = parseInt(data.preview_height || (data.preview && data.preview.height), 10)
     this.previewWidth = parseInt(data.preview_width || (data.preview && data.preview.width), 10)
 
-    this.id = data.id.toString()
+    this.id = data.id ? data.id.toString() : 'No ID available'
     this.tags = getTags(data)
 
     // Too long for conditional

src/structures/SearchParameters.ts

@@ -7,8 +7,14 @@
  * Just an interface for {@link Booru}'s search params :)
  */
 export default interface SearchParameters {
+  /** The limit on *max* posts to show, you might get less posts than this */
   limit?: number
+  /** Should posts be in random order, implementation differs per booru */
   random?: boolean
+  /** Which page of results to fetch */
   page?: number
+  /** The credentials to use to auth with the booru */
   credentials?: any
+  /** Return unavailable posts (ie. banned/deleted posts) */
+  showUnavailable?: boolean
 }