diff --git a/README.md b/README.md index f30fd30a..c5522793 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Internet Archive Search Service -A service for searching and retrieving metadata from the Internet Archive. +A service for searching the Internet Archive. ## Installation ```bash @@ -13,6 +13,7 @@ npm install @internetarchive/search-service ```ts import { SearchService, + SearchType, SortParam, SortDirection } from '@internetarchive/search-service'; @@ -23,50 +24,118 @@ const params = { query: 'collection:books AND title:(goody)', sort: [dateSort], rows: 25, - start: 0, fields: ['identifier', 'collection', 'title', 'creator'] }; -const result = await searchService.performSearch(params); +const result = await searchService.search(params, SearchType.METADATA); if (result.success) { const searchResponse = result.success; - searchResponse.response.numFound // => number - searchResponse.response.docs // => Metadata[] array - searchResponse.response.docs[0].identifier // => 'identifier-foo' + searchResponse.response.totalResults // => number -- total number of search results available to fetch + searchResponse.response.returnedCount // => number -- how many search results are included in this response + searchResponse.response.results // => Result[] array + searchResponse.response.results[0].identifier // => 'some-item-identifier' + searchResponse.response.results[0].title?.value // => 'some-item-title', or possibly undefined if no title exists on the item } ``` -### Fetch Metadata +Currently available search types are `SearchType.METADATA` and `SearchType.FULLTEXT`. -```ts -const metadataResponse: MetadataResponse = await searchService.fetchMetadata('some-identifier'); +### Search parameters -metadataResponse.metadata.identifier // => 'some-identifier' -metadataResponse.metadata.collection.value // => 'some-collection' -metadataResponse.metadata.collection.values // => ['some-collection', 'another-collection', 'more-collections'] -``` +The `params` object passed as first argument to search calls can have the following properties: -## Metadata Values +#### `query` +The full search query, which may include Lucene syntax. -Internet Archive Metadata is expansive and nearly all metadata fields can be returned as either an array, string, or number. +#### `rows` +The maximum number of search results to be retrieved per page. -The Search Service handles all of the possible variations in data formats and converts them to their appropriate types. For instance on date fields, like `date`, it takes the string returned and converts it into a native javascript `Date` value. Similarly for duration-type fields, like `length`, it takes the duration, which can be seconds `324.34` or `hh:mm:ss.ms` and converts them to a `number` in seconds. +#### `page` +Which page of results to retrieve, beginning from page 1. +Each page is sized according to the `rows` parameter, so requesting `{ rows: 20, page: 3 }` +would retrieve results 41-60, etc. -There are parsers for several different field types, like `Number`, `String`, `Date`, and `Duration` and others can be added for other field types. +#### `fields` +An array of metadata field names that should be present on the returned search results. -See `src/models/metadata-fields/field-types.ts` +#### `sort` +An array of sorting parameters to apply to the results. +The first array element specifies the primary sort, the second element the secondary sort, and so on. +Each sorting parameter has the form +```js +{ field: string, direction: 'asc' | 'desc' } +``` +where `field` is the name of the column to sort on (e.g., title) and `direction` is whether to sort ascending or descending. + +#### `aggregations` +An object specifying which aggregations to retrieve with the query. +To retrieve no aggregations at all, this object should be `{ omit: true }`. +To retrieve aggregations for one or more keys, this object should resemble +```js +{ simpleParams: ['subject', 'creator', /*...*/] } +``` -### Usage +To specify the number of buckets for individual aggregation types, the object +should instead use the `advancedParams` property, resembling +```js +{ advancedParams: [{ field: 'subject', size: 2 }, { field: 'creator', size: 4 }, /*...*/] } +``` -```ts -metadata.collection.value // return just the first item of the `values` array, ie. 'my-collection' -metadata.collection.values // returns all values of the array, ie. ['my-collection', 'other-collection'] -metadata.collection.rawValue // return the rawValue. This is useful for inspecting the raw response received. +However, these advanced aggregation parameters are not currently supported by the backend and may be removed at +a later date. + +#### `aggregationsSize` +The number of buckets to be returned for all aggregation types. +This defaults to 6 (the number of facets displayed for each type in the search results sidebar), +but can be overridden using this parameter to retrieve more/fewer buckets as needed. + +#### `pageType` +A string indicating what type of page this data is being requested for. The search backend may +use a different set of default parameters depending on the page type. This defaults to +`'search_results'`, and currently only supports `'search_results' | 'collection_details'`, with +more types to be added in the future. + +#### `pageTarget` +Used in conjunction with `pageType: 'collection_details'` to specify the identifier of the collection +to retrieve results for. + +### Search types + +At present the only two types of search available are Metadata Search (`SearchType.METADATA`) +and Full Text Search (`SearchType.FULLTEXT`). This will eventually be extended to support other +types of search including TV captions and radio transcripts. Calls that do not specify a search +type will default to Metadata Search. + +### Return values + +Calls to `SearchService#search` will return a Promise that either resolves to a `SearchResponse` +object or rejects with a `SearchServiceError`. + +`SearchResponse` objects are structured similar to this example: + +```js +{ + rawResponse: {/*...*/}, // The raw JSON fetched from the server + request: { + clientParameters: {/*...*/}, // The original client parameters sent with the request + finalizedParameters: {/*...*/} // The finalized request parameters as determined by the backend + }, + responseHeader: {/*...*/}, // The header containing info about the response success/failure and processing time + response: { + totalResults: 12345, // The total number of search results matching the query + returnedCount: 50, // The number of search results returned in this response + results: [/*...*/], // The array of search results + aggregations: {/*...*/}, // A record mapping aggregation names to Aggregation objects + schema: {/*...*/} // The data schema to which the returned search results conform + } +} +``` -metadata.date.value // return the date as a javascript `Date` object +### Fetch Metadata -metadata.length.value // return the length (duration) of the item as a number of seconds, can be in the format "hh:mm:ss" or decimal seconds -``` +As of v0.4.0, metadata fetching has been moved to the +[iaux-metadata-service](https://github.com/internetarchive/iaux-metadata-service) package +and is no longer included as part of the Search Service. # Development diff --git a/demo/app-root.ts b/demo/app-root.ts index f0b1e4ce..bf7e89b2 100644 --- a/demo/app-root.ts +++ b/demo/app-root.ts @@ -1,20 +1,12 @@ -import { - css, - CSSResult, - customElement, - html, - internalProperty, - LitElement, - query, - TemplateResult, -} from 'lit-element'; -import { unsafeHTML } from 'lit-html/directives/unsafe-html'; -import { nothing } from 'lit-html'; -import { Metadata } from '../src/models/metadata'; -import { MetadataResponse } from '../src/responses/metadata/metadata-response'; -import { SearchResponse } from '../src/responses/search/search-response'; +import { html, css, LitElement, TemplateResult, CSSResult, nothing } from 'lit'; +import { customElement, query, state } from 'lit/decorators.js'; +import { SearchResponse } from '../src/responses/search-response'; import { SearchService } from '../src/search-service'; import { SearchServiceInterface } from '../src/search-service-interface'; +import { SearchResult } from '../src/models/hit-types/hit'; +import { SearchType } from '../src/search-type'; +import { SearchParams, SortDirection } from '../src/search-params'; +import { Aggregation, Bucket } from '../src/models/aggregation'; @customElement('app-root') export class AppRoot extends LitElement { @@ -23,17 +15,36 @@ export class AppRoot extends LitElement { @query('#search-input') private searchInput!: HTMLInputElement; - @query('#metadata-input') - private metadataInput!: HTMLInputElement; + @query('#debug-info-check') + private debugCheck!: HTMLInputElement; - @internalProperty() + @query('#num-rows') + private rowsInput!: HTMLInputElement; + + @query('#num-aggs') + private numAggsInput!: HTMLInputElement; + + @query(`input[name='sort']:checked`) + private checkedSort!: HTMLInputElement; + + @state() private searchResponse?: SearchResponse; - @internalProperty() - private metadataResponse?: MetadataResponse; + @state() + private aggregationsResponse?: SearchResponse; + + @state() + private loadingSearchResults = false; + + @state() + private loadingAggregations = false; + + private get searchResults(): SearchResult[] | undefined { + return this.searchResponse?.response.results; + } - private get searchResults(): Metadata[] | undefined { - return this.searchResponse?.response.docs; + private get searchAggregations(): Record | undefined { + return this.aggregationsResponse?.response.aggregations; } /** @inheritdoc */ @@ -42,27 +53,125 @@ export class AppRoot extends LitElement {
Search
- + - -
-
- - - + + + + + +
+ Search type: + + + + +
+ +
+ Search size: +
+ + +
+
+ + +
+
+ +
+ Sort by title: + + + + + + +
+ +
+ Include aggregations for: + + + + + + + + + + + + +
${this.searchResults ? this.resultsTemplate : nothing} - ${this.metadataResponse ? this.metadataTemplate : nothing} `; } private get resultsTemplate(): TemplateResult { + return html` + ${this.loadingSearchResults + ? html`

Loading search results...

` + : this.searchResultsTemplate} + ${this.loadingAggregations + ? html`

Loading aggregations...

` + : this.aggregationsTemplate} + `; + } + + private get searchResultsTemplate(): TemplateResult { return html`

Search Results

@@ -70,14 +179,16 @@ export class AppRoot extends LitElement { + ${this.snippetsHeaderTemplate} - ${this.searchResults?.map(metadata => { + ${this.searchResults?.map(hit => { return html` - - + + + ${this.snippetTemplate(hit)} `; })} @@ -86,52 +197,128 @@ export class AppRoot extends LitElement { `; } - private get metadataTemplate(): TemplateResult { - const rawMetadata = this.metadataResponse?.metadata.rawMetadata; - if (!rawMetadata) return html`${nothing}`; - + private get aggregationsTemplate(): TemplateResult { return html` -

Metadata Response

-
Identifier Title
${metadata.identifier}${metadata.title?.value}${hit.identifier}${hit.title?.value ?? '(Untitled)'}
- ${Object.keys(rawMetadata).map( - key => html` - - - - - ` - )} -
${key}${unsafeHTML(rawMetadata[key])}
+
+

Aggregations

+ ${Object.entries(this.searchAggregations ?? {}).map(([key, agg]) => { + return html` +

${key}

+

+ ${agg.buckets + .map((bucket: number | Bucket) => { + if (typeof bucket === 'number') { + return bucket; + } else { + return `${bucket.key} (${bucket.doc_count})`; + } + }) + .join(', ')} +

+ `; + })} +
`; } - async getMetadata(e: Event): Promise { - e.preventDefault(); - const identifier = this.metadataInput.value; - const result = await this.searchService.fetchMetadata(identifier); - this.metadataResponse = result?.success; + private get snippetsHeaderTemplate(): TemplateResult { + return this.searchResults?.some(hit => hit.highlight) + ? html`Snippets` + : html`${nothing}`; } - async search(e: Event): Promise { + private snippetTemplate(hit: SearchResult): TemplateResult { + return hit.highlight + ? html`${hit.highlight.value}` + : html`${nothing}`; + } + + /** + * Conduct a full search (both hits and aggregations) + */ + private async search(e: Event): Promise { e.preventDefault(); const term = this.searchInput.value; + + const checkedSearchType = this.shadowRoot?.querySelector( + `input[name='search-type']:checked` + ) as HTMLInputElement; + + const searchType = + checkedSearchType?.value === 'fts' + ? SearchType.FULLTEXT + : SearchType.METADATA; + + this.fetchSearchResults(term, searchType); + this.fetchAggregations(term, searchType); + } + + /** + * Fetch the search hits + */ + private async fetchSearchResults(query: string, searchType: SearchType) { + const sortParam = + this.checkedSort?.value === 'none' + ? [] + : [ + { + field: 'title', + direction: this.checkedSort?.value as SortDirection, + }, + ]; + + const numRows = Number(this.rowsInput?.value); + const includeDebugging = this.debugCheck?.checked; + + const searchParams: SearchParams = { + query, + rows: numRows, + fields: ['identifier', 'title'], + sort: sortParam, + aggregations: { omit: true }, + debugging: includeDebugging, + }; + + this.loadingSearchResults = true; + const result = await this.searchService.search(searchParams, searchType); + this.loadingSearchResults = false; + + if (result?.success) { + this.searchResponse = result?.success; + } else { + alert(`Oh noes: ${result?.error?.message}`); + console.error('Error searching', result?.error); + } + } + + /** + * Fetch the search aggregations (facets) + */ + private async fetchAggregations(query: string, searchType: SearchType) { + const checkedAggs = this.shadowRoot?.querySelectorAll( + `input[name='aggs']:checked` + ); const aggregations = { - advancedParams: [ - { - field: 'year', - size: 100, - }, - ], + simpleParams: checkedAggs + ? [...checkedAggs].map(elmt => (elmt as HTMLInputElement).value) + : undefined, }; - const searchParams = { - query: term, - rows: 10, - fields: ['identifier', 'title'], + + const numAggs = Number(this.numAggsInput?.value); + + const searchParams: SearchParams = { + query, + rows: 0, aggregations, + aggregationsSize: numAggs, }; - const result = await this.searchService.search(searchParams); + + this.loadingAggregations = true; + const result = await this.searchService.search(searchParams, searchType); + this.loadingAggregations = false; + if (result?.success) { - this.searchResponse = result?.success; + this.aggregationsResponse = result?.success; } else { alert(`Oh noes: ${result?.error?.message}`); console.error('Error searching', result?.error); @@ -140,9 +327,13 @@ export class AppRoot extends LitElement { static get styles(): CSSResult { return css` - /* th { - font-weight: bold; - } */ + .search-options { + margin-top: 0.6rem; + } + + .field-row { + margin: 0.3rem 0; + } `; } } diff --git a/index.ts b/index.ts index 9a244949..290e177e 100644 --- a/index.ts +++ b/index.ts @@ -1,8 +1,12 @@ export { Metadata } from './src/models/metadata'; -export { File } from './src/models/file'; -export { Aggregation, Bucket } from './src/models/aggregation'; -export { Review } from './src/models/review'; -export { SpeechMusicASREntry } from './src/models/speech-music-asr-entry'; +export { ItemHit } from './src/models/hit-types/item-hit'; +export { TextHit } from './src/models/hit-types/text-hit'; +export { SearchResult, HitType } from './src/models/hit-types/hit'; +export { + Aggregation, + AggregationSortType, + Bucket, +} from './src/models/aggregation'; export { DateField } from './src/models/metadata-fields/field-types/date'; @@ -25,14 +29,17 @@ export { MetadataField, } from './src/models/metadata-fields/metadata-field'; -export { MetadataResponse } from './src/responses/metadata/metadata-response'; -export { SearchResponse } from './src/responses/search/search-response'; -export { SearchResponseHeader } from './src/responses/search/search-response-header'; -export { SearchResponseParams } from './src/responses/search/search-response-params'; +export { SearchResponse } from './src/responses/search-response'; +export { SearchResponseHeader } from './src/responses/search-response-header'; +export { SearchResponseParams } from './src/responses/search-response-params'; + +export { MetadataSearchBackend } from './src/search-backend/metadata-search-backend'; +export { FulltextSearchBackend } from './src/search-backend/fulltext-search-backend'; -export { DefaultSearchBackend } from './src/search-backend/default-search-backend'; export { SearchServiceInterface } from './src/search-service-interface'; export { SearchService } from './src/search-service'; +export { SearchBackendOptionsInterface } from './src/search-backend/search-backend-options'; +export { SearchType } from './src/search-type'; export { SearchParams, SortParam, diff --git a/package-lock.json b/package-lock.json index a450bdfc..290d0743 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@internetarchive/search-service", - "version": "0.3.5", + "version": "0.4.0-alpha.6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@internetarchive/search-service", - "version": "0.3.5", + "version": "0.4.0-alpha.6", "license": "AGPL-3.0-only", "dependencies": { "@internetarchive/field-parsers": "^0.1.3", @@ -27,10 +27,10 @@ "eslint-config-prettier": "^6.11.0", "husky": "^1.0.0", "lint-staged": "^10.0.0", - "lit-element": "^2.4.0", - "lit-html": "^1.3.0", + "lit": "^2.2.2", "madge": "^3.12.0", "prettier": "^2.0.4", + "sinon": "^12.0.1", "tslib": "^1.14.1", "typedoc": "^0.20.29", "typescript": "^4.2.2" @@ -534,6 +534,12 @@ "resolved": "https://registry.npmjs.org/@internetarchive/result-type/-/result-type-0.0.1.tgz", "integrity": "sha512-sWahff5oP1xAK1CwAu1/5GTG2RXsdx/sQKn4SSOWH0r0vU2QoX9kAom/jSXeBsmgK0IjTc+9Ty9407SMORi+nQ==" }, + "node_modules/@lit/reactive-element": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.4.1.tgz", + "integrity": "sha512-qDv4851VFSaBWzpS02cXHclo40jsbAjRXnebNXpm0uVg32kCneZPo9RYVQtrTNICtZ+1wAYHu1ZtxWSWMbKrBw==", + "dev": true + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -698,6 +704,41 @@ "node": ">= 8.0.0" } }, + "node_modules/@sinonjs/commons": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", + "integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==", + "dev": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-8.1.0.tgz", + "integrity": "sha512-OAPJUAtgeINhh/TAlUID4QTs53Njm7xzddaVlEs/SXwgtiD1tW22zAB/W1wdqfrpmikgaWQ9Fw6Ws+hsiRm5Vg==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^1.7.0" + } + }, + "node_modules/@sinonjs/samsam": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-6.1.1.tgz", + "integrity": "sha512-cZ7rKJTLiE7u7Wi/v9Hc2fs3Ucc3jrWeMgPHbbTCeVAB2S0wOBbYlkJVeNSL04i7fdhT8wIbDq1zhC/PXTD2SA==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^1.6.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + } + }, + "node_modules/@sinonjs/text-encoding": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", + "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", + "dev": true + }, "node_modules/@types/accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.5.tgz", @@ -990,6 +1031,12 @@ "integrity": "sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==", "dev": true }, + "node_modules/@types/trusted-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz", + "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==", + "dev": true + }, "node_modules/@types/ws": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.0.tgz", @@ -5723,6 +5770,12 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/just-extend": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", + "dev": true + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -6061,6 +6114,17 @@ "node": ">=8" } }, + "node_modules/lit": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/lit/-/lit-2.3.1.tgz", + "integrity": "sha512-TejktDR4mqG3qB32Y8Lm5Lye3c8SUehqz7qRsxe1PqGYL6me2Ef+jeQAEqh20BnnGncv4Yxy2njEIT0kzK1WCw==", + "dev": true, + "dependencies": { + "@lit/reactive-element": "^1.4.0", + "lit-element": "^3.2.0", + "lit-html": "^2.3.0" + } + }, "node_modules/lit-element": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-2.4.0.tgz", @@ -6076,6 +6140,25 @@ "integrity": "sha512-0Q1bwmaFH9O14vycPHw8C/IeHMk/uSDldVLIefu/kfbTBGIc44KGH6A8p1bDfxUfHdc8q6Ct7kQklWoHgr4t1Q==", "dev": true }, + "node_modules/lit/node_modules/lit-element": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz", + "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==", + "dev": true, + "dependencies": { + "@lit/reactive-element": "^1.3.0", + "lit-html": "^2.2.0" + } + }, + "node_modules/lit/node_modules/lit-html": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.3.1.tgz", + "integrity": "sha512-FyKH6LTW6aBdkfNhNSHyZTnLgJSTe5hMk7HFtc/+DcN1w74C215q8B+Cfxc2OuIEpBNcEKxgF64qL8as30FDHA==", + "dev": true, + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, "node_modules/load-json-file": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", @@ -6116,6 +6199,12 @@ "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", "dev": true }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "node_modules/lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -7021,6 +7110,19 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node_modules/nise": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.1.tgz", + "integrity": "sha512-yr5kW2THW1AkxVmCnKEh4nbYkJdB3I7LUkiUgOvEkOp414mc2UMaHMA7pjq1nYowhdoJZGwEKGaQVbxfpWj10A==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^1.8.3", + "@sinonjs/fake-timers": ">=5", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + } + }, "node_modules/node-environment-flags": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", @@ -7481,6 +7583,21 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", "dev": true }, + "node_modules/path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "dependencies": { + "isarray": "0.0.1" + } + }, + "node_modules/path-to-regexp/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true + }, "node_modules/path-type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", @@ -8477,12 +8594,60 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "node_modules/sinon": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-12.0.1.tgz", + "integrity": "sha512-iGu29Xhym33ydkAT+aNQFBINakjq69kKO6ByPvTsm3yyIACfyQttRTP03aBP/I8GfhFmLzrnKwNNkr0ORb1udg==", + "dev": true, + "dependencies": { + "@sinonjs/commons": "^1.8.3", + "@sinonjs/fake-timers": "^8.1.0", + "@sinonjs/samsam": "^6.0.2", + "diff": "^5.0.0", + "nise": "^5.1.0", + "supports-color": "^7.2.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, "node_modules/sinon-chai": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/sinon-chai/-/sinon-chai-3.5.0.tgz", "integrity": "sha512-IifbusYiQBpUxxFJkR3wTU68xzBN0+bxCScEaKMjBvAQERg6FnTTc1F17rseLb1tjmkJ23730AXpFI0c47FgAg==", "dev": true }, + "node_modules/sinon/node_modules/diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/sinon/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/sinon/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -10173,6 +10338,12 @@ "resolved": "https://registry.npmjs.org/@internetarchive/result-type/-/result-type-0.0.1.tgz", "integrity": "sha512-sWahff5oP1xAK1CwAu1/5GTG2RXsdx/sQKn4SSOWH0r0vU2QoX9kAom/jSXeBsmgK0IjTc+9Ty9407SMORi+nQ==" }, + "@lit/reactive-element": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-1.4.1.tgz", + "integrity": "sha512-qDv4851VFSaBWzpS02cXHclo40jsbAjRXnebNXpm0uVg32kCneZPo9RYVQtrTNICtZ+1wAYHu1ZtxWSWMbKrBw==", + "dev": true + }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -10315,6 +10486,41 @@ "picomatch": "^2.2.2" } }, + "@sinonjs/commons": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", + "integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==", + "dev": true, + "requires": { + "type-detect": "4.0.8" + } + }, + "@sinonjs/fake-timers": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-8.1.0.tgz", + "integrity": "sha512-OAPJUAtgeINhh/TAlUID4QTs53Njm7xzddaVlEs/SXwgtiD1tW22zAB/W1wdqfrpmikgaWQ9Fw6Ws+hsiRm5Vg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0" + } + }, + "@sinonjs/samsam": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-6.1.1.tgz", + "integrity": "sha512-cZ7rKJTLiE7u7Wi/v9Hc2fs3Ucc3jrWeMgPHbbTCeVAB2S0wOBbYlkJVeNSL04i7fdhT8wIbDq1zhC/PXTD2SA==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.6.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + } + }, + "@sinonjs/text-encoding": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", + "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", + "dev": true + }, "@types/accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.5.tgz", @@ -10607,6 +10813,12 @@ "integrity": "sha512-dIPoZ3g5gcx9zZEszaxLSVTvMReD3xxyyDnQUjA6IYDG9Ba2AV0otMPs+77sG9ojB4Qr2N2Vk5RnKeuA0X/0bg==", "dev": true }, + "@types/trusted-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz", + "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==", + "dev": true + }, "@types/ws": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.0.tgz", @@ -14316,6 +14528,12 @@ "universalify": "^2.0.0" } }, + "just-extend": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", + "dev": true + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -14615,6 +14833,38 @@ } } }, + "lit": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/lit/-/lit-2.3.1.tgz", + "integrity": "sha512-TejktDR4mqG3qB32Y8Lm5Lye3c8SUehqz7qRsxe1PqGYL6me2Ef+jeQAEqh20BnnGncv4Yxy2njEIT0kzK1WCw==", + "dev": true, + "requires": { + "@lit/reactive-element": "^1.4.0", + "lit-element": "^3.2.0", + "lit-html": "^2.3.0" + }, + "dependencies": { + "lit-element": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-3.2.2.tgz", + "integrity": "sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==", + "dev": true, + "requires": { + "@lit/reactive-element": "^1.3.0", + "lit-html": "^2.2.0" + } + }, + "lit-html": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-2.3.1.tgz", + "integrity": "sha512-FyKH6LTW6aBdkfNhNSHyZTnLgJSTe5hMk7HFtc/+DcN1w74C215q8B+Cfxc2OuIEpBNcEKxgF64qL8as30FDHA==", + "dev": true, + "requires": { + "@types/trusted-types": "^2.0.2" + } + } + } + }, "lit-element": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-2.4.0.tgz", @@ -14664,6 +14914,12 @@ "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", "dev": true }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -15385,6 +15641,19 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "nise": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.1.tgz", + "integrity": "sha512-yr5kW2THW1AkxVmCnKEh4nbYkJdB3I7LUkiUgOvEkOp414mc2UMaHMA7pjq1nYowhdoJZGwEKGaQVbxfpWj10A==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.8.3", + "@sinonjs/fake-timers": ">=5", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + } + }, "node-environment-flags": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", @@ -15759,6 +16028,23 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", "dev": true }, + "path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "requires": { + "isarray": "0.0.1" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true + } + } + }, "path-type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", @@ -16516,6 +16802,43 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "sinon": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-12.0.1.tgz", + "integrity": "sha512-iGu29Xhym33ydkAT+aNQFBINakjq69kKO6ByPvTsm3yyIACfyQttRTP03aBP/I8GfhFmLzrnKwNNkr0ORb1udg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.8.3", + "@sinonjs/fake-timers": "^8.1.0", + "@sinonjs/samsam": "^6.0.2", + "diff": "^5.0.0", + "nise": "^5.1.0", + "supports-color": "^7.2.0" + }, + "dependencies": { + "diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "sinon-chai": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/sinon-chai/-/sinon-chai-3.5.0.tgz", diff --git a/package.json b/package.json index 20f4818e..305c6a3b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@internetarchive/search-service", - "version": "0.3.5", + "version": "0.4.0", "description": "A search service for the Internet Archive", "license": "AGPL-3.0-only", "main": "dist/index.js", @@ -38,10 +38,10 @@ "eslint-config-prettier": "^6.11.0", "husky": "^1.0.0", "lint-staged": "^10.0.0", - "lit-element": "^2.4.0", - "lit-html": "^1.3.0", + "lit": "^2.2.2", "madge": "^3.12.0", "prettier": "^2.0.4", + "sinon": "^12.0.1", "tslib": "^1.14.1", "typedoc": "^0.20.29", "typescript": "^4.2.2" diff --git a/src/models/aggregation.ts b/src/models/aggregation.ts index b5ff10db..512ebb5f 100644 --- a/src/models/aggregation.ts +++ b/src/models/aggregation.ts @@ -1,3 +1,5 @@ +import { Memoize } from 'typescript-memoize'; + /** * A Bucket is an individual entry for a facet or bin for a histogram * @@ -9,22 +11,86 @@ export interface Bucket { doc_count: number; } -/** - * Aggregations are the datasource for facets and histograms - * - * @export - * @interface Aggregation - */ -export interface Aggregation { - doc_count_error_upper_bound?: number; - sum_other_doc_count?: number; +export enum AggregationSortType { /** - * The year_histogram returns a `number` array, and - * other facets return a `Bucket` array + * Sort descending numerically by count/frequency + */ + COUNT, + /** + * Sort ascending alphabetically by key */ + ALPHABETICAL, +} + +export interface AggregationOptions { buckets: Bucket[] | number[]; + doc_count_error_upper_bound?: number; + sum_other_doc_count?: number; first_bucket_key?: number; last_bucket_key?: number; number_buckets?: number; interval?: number; } + +/** + * Aggregations are the datasource for facets and histograms + */ +export class Aggregation { + /** + * The year_histogram returns a `number` array, and + * other facets return a `Bucket` array. + */ + readonly buckets: Bucket[] | number[]; + readonly doc_count_error_upper_bound?: number; + readonly sum_other_doc_count?: number; + readonly first_bucket_key?: number; + readonly last_bucket_key?: number; + readonly number_buckets?: number; + readonly interval?: number; + + constructor(options: AggregationOptions) { + this.buckets = options.buckets; + this.doc_count_error_upper_bound = options.doc_count_error_upper_bound; + this.sum_other_doc_count = options.sum_other_doc_count; + this.first_bucket_key = options.first_bucket_key; + this.last_bucket_key = options.last_bucket_key; + this.number_buckets = options.number_buckets; + this.interval = options.interval; + } + + /** + * Returns a sorted copy of this aggregation's buckets, according to the given sort option. + * This method is memoized and repeat calls should not incur a performance cost for a given + * sort type after the first such call per instance. + * + * Purely numeric buckets (e.g., year_histogram) are not sorted by this method and are + * returned as-is. + * + * @param sortType What to sort the buckets on. + * Accepted values are `AggregationSortType.COUNT` (descending order) and + * `AggregationSortType.ALPHABETICAL` (ascending order). + */ + @Memoize() + getSortedBuckets(sortType?: AggregationSortType): Bucket[] | number[] { + // Don't apply sorts to numeric buckets. + // Assumption here is that all the buckets have the same type as the + // first bucket (which should be true in principle). + if (typeof this.buckets[0] === 'number') { + return [...(this.buckets as number[])]; + } + + // Default locale & collation options + const collator = new Intl.Collator(); + + switch (sortType) { + case AggregationSortType.ALPHABETICAL: + return [...(this.buckets as Bucket[])].sort((a, b) => { + return collator.compare(a.key.toString(), b.key.toString()); + }); + case AggregationSortType.COUNT: + default: + // Sorted by count by default + return [...(this.buckets as Bucket[])]; + } + } +} diff --git a/src/models/file.ts b/src/models/file.ts deleted file mode 100644 index c1cc4936..00000000 --- a/src/models/file.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Memoize } from 'typescript-memoize'; -import { - Byte, - ByteParser, - Duration, - DurationParser, - NumberParser, -} from '@internetarchive/field-parsers'; - -/** - * This represents an Internet Archive File - * - * @export - * @class File - */ -export class File { - rawValue: Record; - - get name(): string { - return this.rawValue.name; - } - - get source(): string { - return this.rawValue.source; - } - - get btih(): string { - return this.rawValue.btih; - } - - get md5(): string { - return this.rawValue.md5; - } - - get format(): string { - return this.rawValue.format; - } - - get mtime(): string { - return this.rawValue.mtime; - } - - get crc32(): string { - return this.rawValue.crc32; - } - - get sha1(): string { - return this.rawValue.sha1; - } - - get original(): string | undefined { - return this.rawValue.original; - } - - @Memoize() get size(): Byte | undefined { - return this.rawValue.size - ? ByteParser.shared.parseValue(this.rawValue.size) - : undefined; - } - - get title(): string | undefined { - return this.rawValue.title; - } - - @Memoize() get length(): Duration | undefined { - return this.rawValue.length - ? DurationParser.shared.parseValue(this.rawValue.length) - : undefined; - } - - @Memoize() get height(): number | undefined { - return this.rawValue.height - ? NumberParser.shared.parseValue(this.rawValue.height) - : undefined; - } - - @Memoize() get width(): number | undefined { - return this.rawValue.width - ? NumberParser.shared.parseValue(this.rawValue.width) - : undefined; - } - - @Memoize() get track(): number | undefined { - return this.rawValue.track - ? NumberParser.shared.parseValue(this.rawValue.track) - : undefined; - } - - get external_identifier(): string | string[] | undefined { - return this.rawValue.external_identifier; - } - - get creator(): string | undefined { - return this.rawValue.creator; - } - - get album(): string | undefined { - return this.rawValue.album; - } - - constructor(json: Record) { - this.rawValue = json; - } -} diff --git a/src/models/hit-types/hit.ts b/src/models/hit-types/hit.ts new file mode 100644 index 00000000..03e185b0 --- /dev/null +++ b/src/models/hit-types/hit.ts @@ -0,0 +1,23 @@ +import type { ItemHit } from './item-hit'; +import type { TextHit } from './text-hit'; + +/** + * Union of the different hit_type values returned by the PPS. + * There will probably be more of these. + */ +export type HitType = 'item' | 'text'; + +/** + * Applying this to the Result type forces Intellisense to present Result as + * a type in its own right, and not as the underlying merge type it aliases. + * Really just to keep things clean at the call site. + */ +interface PreserveAlias {} // eslint-disable-line @typescript-eslint/no-empty-interface + +/** + * Result is an expansive type definition encompassing all the optional + * and required properties that may occur on any type of search result + * ('hit') returned by the various search backends. (Most metadata + * properties are optional anyway). + */ +export type SearchResult = Partial & PreserveAlias; diff --git a/src/models/hit-types/item-hit.ts b/src/models/hit-types/item-hit.ts new file mode 100644 index 00000000..3938329c --- /dev/null +++ b/src/models/hit-types/item-hit.ts @@ -0,0 +1,297 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Memoize } from 'typescript-memoize'; +import type { Metadata } from '../metadata'; +import { BooleanField } from '../metadata-fields/field-types/boolean'; +import { DateField } from '../metadata-fields/field-types/date'; +import { MediaTypeField } from '../metadata-fields/field-types/mediatype'; +import { NumberField } from '../metadata-fields/field-types/number'; +import { StringField } from '../metadata-fields/field-types/string'; + +/** + * A model that describes an item hit from a Metadata Search via the PPS endpoint. + * + * The fields in here are cast to their respective field types. See `metadata-fields/field-type`. + * + * Add additional fields as needed. + * + * @export + * @class ItemHit + */ +export class ItemHit { + /** + * This is the raw hit response; useful for inspecting the raw data + * returned from the server. + */ + rawMetadata?: Record; + + constructor(json: Record) { + this.rawMetadata = json; + } + + /** + * The item identifier. + * + * _Note_: This is a plain string instead of a `MetadataField` since it's + * the primary key of the item. + */ + get identifier(): typeof Metadata.prototype.identifier { + return this.rawMetadata?.fields?.identifier; + } + + /** + * May be a superset of metadata collection field. + * Multivalued. + */ + @Memoize() get collection(): typeof Metadata.prototype.collection { + return this.rawMetadata?.fields?.collection + ? new StringField(this.rawMetadata.fields.collection) + : undefined; + } + + /** + * Computed during document construction, for collection items only. + * Optional. + */ + @Memoize() get collection_files_count(): NumberField | undefined { + return this.rawMetadata?.fields?.collection_files_count + ? new NumberField(this.rawMetadata.fields.collection_files_count) + : undefined; + } + + /** + * In bytes; computed during document construction, for collection items only. + * Optional. + */ + @Memoize() get collection_size(): typeof Metadata.prototype.collection_size { + return this.rawMetadata?.fields?.collection_size + ? new NumberField(this.rawMetadata.fields.collection_size) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get creator(): typeof Metadata.prototype.creator { + return this.rawMetadata?.fields?.creator + ? new StringField(this.rawMetadata.fields.creator) + : undefined; + } + + /** + * Optional. + */ + @Memoize() get date(): typeof Metadata.prototype.date { + return this.rawMetadata?.fields?.date + ? new DateField(this.rawMetadata.fields.date) + : undefined; + } + + /** Optional. */ + @Memoize() get description(): typeof Metadata.prototype.description { + return this.rawMetadata?.fields?.description + ? new StringField(this.rawMetadata.fields.description) + : undefined; + } + + /** + * Total views over item lifetime, updated by audit consultation with Views API. + * Optional. + */ + @Memoize() get downloads(): typeof Metadata.prototype.downloads { + return this.rawMetadata?.fields?.downloads + ? new NumberField(this.rawMetadata.fields.downloads) + : undefined; + } + + /** + * Computed during document construction. + */ + @Memoize() get files_count(): typeof Metadata.prototype.files_count { + return this.rawMetadata?.fields?.files_count + ? new NumberField(this.rawMetadata.fields.files_count) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get genre(): StringField | undefined { + return this.rawMetadata?.fields?.genre + ? new StringField(this.rawMetadata.fields.genre) + : undefined; + } + + /** + * Item characterization including noindex status. + * Multivalued. + */ + @Memoize() get indexflag(): StringField | undefined { + return this.rawMetadata?.fields?.indexflag + ? new StringField(this.rawMetadata.fields.indexflag) + : undefined; + } + + /** + * In bytes; computed during document construction. + */ + @Memoize() get item_size(): typeof Metadata.prototype.item_size { + return this.rawMetadata?.fields?.item_size + ? new NumberField(this.rawMetadata.fields.item_size) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get language(): typeof Metadata.prototype.language { + return this.rawMetadata?.fields?.language + ? new StringField(this.rawMetadata.fields.language) + : undefined; + } + + /** + * May be stale. + * Optional. + */ + @Memoize() get lending___available_to_borrow(): BooleanField | undefined { + return this.rawMetadata?.fields?.lending___available_to_borrow != null + ? new BooleanField(this.rawMetadata.fields.lending___available_to_borrow) + : undefined; + } + + /** + * May be stale. + * Optional. + */ + @Memoize() get lending___available_to_browse(): BooleanField | undefined { + return this.rawMetadata?.fields?.lending___available_to_browse != null + ? new BooleanField(this.rawMetadata.fields.lending___available_to_browse) + : undefined; + } + + /** + * May be stale. + * Optional. + */ + @Memoize() get lending___available_to_waitlist(): BooleanField | undefined { + return this.rawMetadata?.fields?.lending___available_to_waitlist != null + ? new BooleanField( + this.rawMetadata.fields.lending___available_to_waitlist + ) + : undefined; + } + + /** + * May be stale. + * Optional. + */ + @Memoize() get lending___status(): StringField | undefined { + return this.rawMetadata?.fields?.lending___status + ? new StringField(this.rawMetadata.fields.lending___status) + : undefined; + } + + /** Optional. */ + @Memoize() get licenseurl(): StringField | undefined { + return this.rawMetadata?.fields?.licenseurl + ? new StringField(this.rawMetadata.fields.licenseurl) + : undefined; + } + + @Memoize() get mediatype(): typeof Metadata.prototype.mediatype { + return this.rawMetadata?.fields?.mediatype + ? new MediaTypeField(this.rawMetadata.fields.mediatype) + : undefined; + } + + /** + * Views over a month, updated by audit consultation with Views API. + * Optional. + */ + @Memoize() get month(): typeof Metadata.prototype.month { + return this.rawMetadata?.fields?.month + ? new NumberField(this.rawMetadata.fields.month) + : undefined; + } + + /** Optional. */ + @Memoize() get noindex(): typeof Metadata.prototype.noindex { + return this.rawMetadata?.fields?.noindex != null + ? new BooleanField(this.rawMetadata.fields.noindex) + : undefined; + } + + /** + * Computed during document construction. + * Optional. + */ + @Memoize() get num_favorites(): typeof Metadata.prototype.num_favorites { + return this.rawMetadata?.fields?.num_favorites + ? new NumberField(this.rawMetadata.fields.num_favorites) + : undefined; + } + + /** + * Computed during document construction. + * Optional. + */ + @Memoize() get num_reviews(): typeof Metadata.prototype.num_reviews { + return this.rawMetadata?.fields?.num_reviews + ? new NumberField(this.rawMetadata.fields.num_reviews) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get subject(): StringField | undefined { + return this.rawMetadata?.fields?.subject + ? new StringField(this.rawMetadata.fields.subject) + : undefined; + } + + /** Optional. */ + @Memoize() get title(): typeof Metadata.prototype.title { + return this.rawMetadata?.fields?.title + ? new StringField(this.rawMetadata.fields.title) + : undefined; + } + + /** Optional. */ + @Memoize() get type(): typeof Metadata.prototype.type { + return this.rawMetadata?.fields?.type + ? new StringField(this.rawMetadata.fields.type) + : undefined; + } + + /** Optional. */ + @Memoize() get volume(): typeof Metadata.prototype.volume { + return this.rawMetadata?.fields?.volume + ? new StringField(this.rawMetadata.fields.volume) + : undefined; + } + + /** + * Views over a seven-day period, updated by audit consultation with Views API. + * Optional. + */ + @Memoize() get week(): typeof Metadata.prototype.week { + return this.rawMetadata?.fields?.week + ? new NumberField(this.rawMetadata.fields.week) + : undefined; + } + + /** + * Computed from date. + * Optional. + */ + @Memoize() get year(): NumberField | undefined { + return this.rawMetadata?.fields?.year + ? new NumberField(this.rawMetadata.fields.year) + : undefined; + } +} diff --git a/src/models/hit-types/text-hit.ts b/src/models/hit-types/text-hit.ts new file mode 100644 index 00000000..c40de2a0 --- /dev/null +++ b/src/models/hit-types/text-hit.ts @@ -0,0 +1,199 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Memoize } from 'typescript-memoize'; +import type { Metadata } from '../metadata'; +import { BooleanField } from '../metadata-fields/field-types/boolean'; +import { DateField } from '../metadata-fields/field-types/date'; +import { MediaTypeField } from '../metadata-fields/field-types/mediatype'; +import { NumberField } from '../metadata-fields/field-types/number'; +import { StringField } from '../metadata-fields/field-types/string'; + +/** + * A model that describes a textual hit from a full text search via the PPS endpoint. + * + * The fields in here are cast to their respective field types. See `metadata-fields/field-type`. + * + * Add additional fields as needed. + * + * @export + * @class TextHit + */ +export class TextHit { + /** + * This is the raw hit response; useful for inspecting the raw data + * returned from the server. + */ + rawMetadata?: Record; + + constructor(json: Record) { + this.rawMetadata = json; + } + + /** + * The item identifier. + * + * _Note_: This is a plain string instead of a `MetadataField` since it's + * the primary key of the item. + */ + get identifier(): typeof Metadata.prototype.identifier { + return this.rawMetadata?.fields?.identifier; + } + + /** + * Synthesized in processing of FTS API hit; TBD + * Optional. + */ + @Memoize() get highlight(): StringField | undefined { + // Note: _not_ inside the fields object. + return this.rawMetadata?.highlight?.text + ? new StringField(this.rawMetadata.highlight.text) + : undefined; + } + + /** Multivalued. */ + @Memoize() get collection(): typeof Metadata.prototype.collection { + return this.rawMetadata?.fields?.collection + ? new StringField(this.rawMetadata.fields.collection) + : undefined; + } + + @Memoize() get created_on(): DateField | undefined { + return this.rawMetadata?.fields?.created_on + ? new DateField(this.rawMetadata.fields.created_on) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get creator(): typeof Metadata.prototype.creator { + return this.rawMetadata?.fields?.creator + ? new StringField(this.rawMetadata.fields.creator) + : undefined; + } + + /** Optional. */ + @Memoize() get date(): typeof Metadata.prototype.date { + return this.rawMetadata?.fields?.date + ? new DateField(this.rawMetadata.fields.date) + : undefined; + } + + /** + * Contents of hilighting in FTS API hit, TBD + * Optional. + * Multivalued. + */ + @Memoize() get description(): typeof Metadata.prototype.description { + return this.rawMetadata?.fields?.description + ? new StringField(this.rawMetadata.fields.description) + : undefined; + } + + /** + * Total views over ITEM (not text) lifetime, updated by audit consultation with Views API. + * Optional. + */ + @Memoize() get downloads(): typeof Metadata.prototype.downloads { + return this.rawMetadata?.fields?.downloads + ? new NumberField(this.rawMetadata.fields.downloads) + : undefined; + } + + @Memoize() get filename(): StringField | undefined { + return this.rawMetadata?.fields?.filename + ? new StringField(this.rawMetadata.fields.filename) + : undefined; + } + + @Memoize() get file_basename(): StringField | undefined { + return this.rawMetadata?.fields?.file_basename + ? new StringField(this.rawMetadata.fields.file_basename) + : undefined; + } + + @Memoize() get file_creation_mtime(): NumberField | undefined { + return this.rawMetadata?.fields?.file_creation_mtime + ? new NumberField(this.rawMetadata.fields.file_creation_mtime) + : undefined; + } + + @Memoize() get mediatype(): typeof Metadata.prototype.mediatype { + return this.rawMetadata?.fields?.mediatype + ? new MediaTypeField(this.rawMetadata.fields.mediatype) + : undefined; + } + + /** Optional. */ + @Memoize() get page_num(): NumberField | undefined { + return this.rawMetadata?.fields?.page_num + ? new NumberField(this.rawMetadata.fields.page_num) + : undefined; + } + + /** Optional. */ + @Memoize() get publicdate(): typeof Metadata.prototype.publicdate { + return this.rawMetadata?.fields?.publicdate + ? new DateField(this.rawMetadata.fields.publicdate) + : undefined; + } + + /** + * Computed in processing of FTS API hit. + */ + @Memoize() get result_in_subfile(): BooleanField | undefined { + return this.rawMetadata?.fields?.result_in_subfile != null + ? new BooleanField(this.rawMetadata.fields.result_in_subfile) + : undefined; + } + + /** Optional. */ + @Memoize() get reviewdate(): typeof Metadata.prototype.reviewdate { + return this.rawMetadata?.fields?.reviewdate + ? new DateField(this.rawMetadata.fields.reviewdate) + : undefined; + } + + /** + * Optional. + * Multivalued. + */ + @Memoize() get subject(): StringField | undefined { + return this.rawMetadata?.fields?.subject + ? new StringField(this.rawMetadata.fields.subject) + : undefined; + } + + /** Optional. */ + @Memoize() get title(): typeof Metadata.prototype.title { + return this.rawMetadata?.fields?.title + ? new StringField(this.rawMetadata.fields.title) + : undefined; + } + + @Memoize() get updated_on(): DateField | undefined { + return this.rawMetadata?.fields?.updated_on + ? new DateField(this.rawMetadata.fields.updated_on) + : undefined; + } + + /** + * Computed from date. + * Optional. + */ + @Memoize() get year(): NumberField | undefined { + return this.rawMetadata?.fields?.year + ? new NumberField(this.rawMetadata.fields.year) + : undefined; + } + + /** + * Synthesized in processing of FTS API hit; TBD + * Optional. + */ + @Memoize() get __href__(): StringField | undefined { + return this.rawMetadata?.fields?.__href__ + ? new StringField(this.rawMetadata.fields.__href__) + : undefined; + } +} diff --git a/src/models/metadata.ts b/src/models/metadata.ts index b6730609..00009a24 100644 --- a/src/models/metadata.ts +++ b/src/models/metadata.ts @@ -260,7 +260,7 @@ export class Metadata { } @Memoize() get noindex(): BooleanField | undefined { - return this.rawMetadata?.noindex + return this.rawMetadata?.noindex != null ? new BooleanField(this.rawMetadata.noindex) : undefined; } @@ -349,12 +349,6 @@ export class Metadata { : undefined; } - @Memoize() get snippets(): StringField | undefined { - return this.rawMetadata?.snippets - ? new StringField(this.rawMetadata.snippets) - : undefined; - } - @Memoize() get source(): StringField | undefined { return this.rawMetadata?.source ? new StringField(this.rawMetadata.source) diff --git a/src/models/review.ts b/src/models/review.ts deleted file mode 100644 index 9ab3eacd..00000000 --- a/src/models/review.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Memoize } from 'typescript-memoize'; -import { DateParser, NumberParser } from '@internetarchive/field-parsers'; - -export class Review { - rawValue: Record; - - get reviewbody(): string | undefined { - return this.rawValue.reviewbody; - } - - get reviewtitle(): string | undefined { - return this.rawValue.reviewtitle; - } - - get reviewer(): string | undefined { - return this.rawValue.reviewer; - } - - @Memoize() get reviewdate(): Date | undefined { - return this.rawValue.reviewdate - ? DateParser.shared.parseValue(this.rawValue.reviewdate) - : undefined; - } - - @Memoize() get createdate(): Date | undefined { - return this.rawValue.createdate - ? DateParser.shared.parseValue(this.rawValue.createdate) - : undefined; - } - - @Memoize() get stars(): number | undefined { - return this.rawValue.stars - ? NumberParser.shared.parseValue(this.rawValue.stars) - : undefined; - } - - constructor(json: Record) { - this.rawValue = json; - } -} diff --git a/src/models/speech-music-asr-entry.ts b/src/models/speech-music-asr-entry.ts deleted file mode 100644 index b73b4a47..00000000 --- a/src/models/speech-music-asr-entry.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * This is the format used for radio transcripts - */ -export interface SpeechMusicASREntry { - end: number; - id: number; - is_music: boolean; - start: number; - text: string; -} diff --git a/src/responses/metadata/metadata-response.ts b/src/responses/metadata/metadata-response.ts deleted file mode 100644 index 669f5720..00000000 --- a/src/responses/metadata/metadata-response.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { File } from '../../models/file'; -import { Metadata } from '../../models/metadata'; -import { Review } from '../../models/review'; -import { SpeechMusicASREntry } from '../../models/speech-music-asr-entry'; - -/** - * The main top-level reponse when fetching Metadata - * - * @export - * @class MetadataResponse - */ -export class MetadataResponse { - rawResponse: any; - - created: number; - - d1: string; - - d2: string; - - dir: string; - - files: File[]; - - files_count: number; - - item_last_updated: number; - - item_size: number; - - metadata: Metadata; - - server: string; - - uniq: number; - - workable_servers: string[]; - - speech_vs_music_asr?: SpeechMusicASREntry[]; - - reviews?: Review[]; - - constructor(json: Record) { - this.rawResponse = json; - this.created = json.created; - this.d1 = json.d1; - this.d2 = json.d2; - this.dir = json.dir; - this.files = json.files?.map((file: any) => new File(file)); - this.files_count = json.files_count; - this.item_last_updated = json.item_last_updated; - this.item_size = json.item_size; - this.metadata = new Metadata(json.metadata); - this.server = json.server; - this.uniq = json.uniq; - this.workable_servers = json.workable_servers; - this.speech_vs_music_asr = json.speech_vs_music_asr; - this.reviews = json.reviews?.map((entry: any) => new Review(entry)); - } -} diff --git a/src/responses/search-hit-schema.ts b/src/responses/search-hit-schema.ts new file mode 100644 index 00000000..7f3bd2e2 --- /dev/null +++ b/src/responses/search-hit-schema.ts @@ -0,0 +1,28 @@ +import { HitType } from '../models/hit-types/hit'; + +/** + * Schema for individual field types within a hit + */ +export interface FieldSchema { + mapping: string; + multivalue: boolean; + optional?: boolean; + comment?: string; +} + +/** + * Top-level schema structure returned by the PPS + */ +export interface SearchHitSchema { + /** + * A string identifying what type of hits were returned + * (and which this schema describes) + */ + hit_type: HitType; + + /** + * A map of the fields present on each search hit, with info + * about their type and whether they are optional or multivalued. + */ + field_properties: Record; +} diff --git a/src/responses/search-request-params.ts b/src/responses/search-request-params.ts new file mode 100644 index 00000000..0671de2a --- /dev/null +++ b/src/responses/search-request-params.ts @@ -0,0 +1,23 @@ +/** + * The server-parsed request parameters that are returned + * with responses from the PPS endpoint. + * + * While similar in structure to the `SearchParams` passed + * into a `search` call, this interface describes the _parsed_ + * parameters as the server has interpreted them (e.g., with + * raw sort and aggregation strings rather than the structured + * objects that callers pass with a query). + */ +export interface SearchRequestParams { + client?: string; + user_query?: string; + service_backend?: string; + page_type?: string; + page_target?: string; + page?: number; + hits_per_page?: number; + fields?: string[]; + sort?: string[]; + aggregations?: string[]; + aggregations_size?: number; +} diff --git a/src/responses/search-request.ts b/src/responses/search-request.ts new file mode 100644 index 00000000..73eddd2b --- /dev/null +++ b/src/responses/search-request.ts @@ -0,0 +1,22 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { SearchRequestParams } from './search-request-params'; + +/** + * A model for the request parameters returned with each search response. + */ +export class SearchRequest { + /** + * The original client parameters sent with the request + */ + clientParameters: SearchRequestParams; + + /** + * The finalized request parameters as determined by the backend + */ + finalizedParameters: SearchRequestParams; + + constructor(json: Record) { + this.clientParameters = json.client_parameters; + this.finalizedParameters = json.finalized_parameters; + } +} diff --git a/src/responses/search-response-details.ts b/src/responses/search-response-details.ts new file mode 100644 index 00000000..edc94fb5 --- /dev/null +++ b/src/responses/search-response-details.ts @@ -0,0 +1,100 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Aggregation } from '../models/aggregation'; +import { SearchResult, HitType } from '../models/hit-types/hit'; +import { ItemHit } from '../models/hit-types/item-hit'; +import { TextHit } from '../models/hit-types/text-hit'; +import type { SearchHitSchema } from './search-hit-schema'; + +/** + * The structure of the response body returned from the PPS endpoint. + */ +export interface SearchResponseBody { + hits: SearchResponseHits; + aggregations?: Record; +} + +/** + * The structure of the response body `hits` object returned from the PPS endpoint. + */ +export interface SearchResponseHits { + total: number; + returned: number; + hits: Record[]; +} + +/** + * This is the search response details inside the SearchResponse object that contains + * the search results, under the `results` key. + * + * @export + */ +export class SearchResponseDetails { + /** + * Total number of results found + */ + totalResults: number; + + /** + * Number of returned hits in this response + */ + returnedCount: number; + + /** + * The array of search results + */ + results: SearchResult[]; + + /** + * Requested aggregations such as facets or histogram data + * + * @type {Record} + * @memberof SearchResponseDetails + */ + aggregations?: Record; + + /** + * The hit schema for this response + */ + schema?: SearchHitSchema; + + constructor(body: SearchResponseBody, schema: SearchHitSchema) { + this.schema = schema; + const hitType = schema?.hit_type; + + this.totalResults = body?.hits?.total ?? 0; + this.returnedCount = body?.hits?.returned ?? 0; + this.results = + body?.hits?.hits?.map((hit: SearchResult) => + SearchResponseDetails.createResult(hitType, hit) + ) ?? []; + + // Construct Aggregation objects + if (body?.aggregations) { + this.aggregations = Object.entries(body.aggregations).reduce( + (acc, [key, val]) => { + acc[key] = new Aggregation(val); + return acc; + }, + {} as Record + ); + } + } + + /** + * Returns a correctly-typed search result depending on the schema's hit_type. + */ + private static createResult( + type: HitType, + result: SearchResult + ): SearchResult { + switch (type) { + case 'item': + return new ItemHit(result); + case 'text': + return new TextHit(result); + default: + // The hit type doesn't tell us what to construct, so just return the input as-is + return result; + } + } +} diff --git a/src/responses/search-response-header.ts b/src/responses/search-response-header.ts new file mode 100644 index 00000000..df245021 --- /dev/null +++ b/src/responses/search-response-header.ts @@ -0,0 +1,4 @@ +export interface SearchResponseHeader { + succeeded: boolean; + query_time: number; +} diff --git a/src/responses/search/search-response-params.ts b/src/responses/search-response-params.ts similarity index 100% rename from src/responses/search/search-response-params.ts rename to src/responses/search-response-params.ts diff --git a/src/responses/search/search-response.ts b/src/responses/search-response.ts similarity index 64% rename from src/responses/search/search-response.ts rename to src/responses/search-response.ts index 20c5c8da..bd68d927 100644 --- a/src/responses/search/search-response.ts +++ b/src/responses/search-response.ts @@ -1,9 +1,10 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { SearchResponseHeader } from './search-response-header'; import { SearchResponseDetails } from './search-response-details'; +import { SearchRequest } from './search-request'; /** - * The top-level response model when retrieving a response from the advanced search endpoint. + * The top-level response model when retrieving a response from the page production service endpoint. * * @export * @class SearchResponse @@ -18,7 +19,13 @@ export class SearchResponse { rawResponse: Record; /** - * The resonse header + * The request object returned by the backend, specifying the query parameters and + * how the backend interpreted them. + */ + request: SearchRequest; + + /** + * The response header * * @type {SearchResponseHeader} * @memberof SearchResponse @@ -35,7 +42,11 @@ export class SearchResponse { constructor(json: Record) { this.rawResponse = json; - this.responseHeader = json.responseHeader; - this.response = new SearchResponseDetails(json.response); + this.request = new SearchRequest(json.request); + this.responseHeader = json.response?.header; + this.response = new SearchResponseDetails( + json.response?.body, + json.response?.hit_schema + ); } } diff --git a/src/responses/search/search-response-details.ts b/src/responses/search/search-response-details.ts deleted file mode 100644 index 0d62adb3..00000000 --- a/src/responses/search/search-response-details.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Aggregation } from '../../models/aggregation'; -import { Metadata } from '../../models/metadata'; - -/** - * This is the search response details inside the SearchResponse object that contains - * the search results, under the `docs` key. - * - * @export - * @class Response - */ -export class SearchResponseDetails { - /** - * Total number of results found - * - * @type {number} - * @memberof Response - */ - numFound: number; - - /** - * Search result start number for pagination - * - * @type {number} - * @memberof Response - */ - start: number; - - /** - * The array of search results - * - * @type {Metadata[]} - * @memberof Response - */ - docs: Metadata[]; - - /** - * Requested aggregations such as facets or histogram data - * - * @type {Record} - * @memberof SearchResponseDetails - */ - aggregations?: Record; - - constructor(json: SearchResponseDetails) { - this.numFound = json.numFound; - this.start = json.start; - this.docs = json.docs.map((doc: any) => new Metadata(doc)); - this.aggregations = json.aggregations; - } -} diff --git a/src/responses/search/search-response-header.ts b/src/responses/search/search-response-header.ts deleted file mode 100644 index 9d2a148c..00000000 --- a/src/responses/search/search-response-header.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { SearchResponseParams } from './search-response-params'; - -export interface SearchResponseHeader { - status: number; - QTime: number; - params: SearchResponseParams; -} diff --git a/src/search-backend/default-search-backend.ts b/src/search-backend/base-search-backend.ts similarity index 66% rename from src/search-backend/default-search-backend.ts rename to src/search-backend/base-search-backend.ts index 71885802..6e897a6d 100644 --- a/src/search-backend/default-search-backend.ts +++ b/src/search-backend/base-search-backend.ts @@ -1,28 +1,28 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { SearchBackendInterface } from './search-backend-interface'; -import { SearchParams } from '../search-params'; -import { Result } from '@internetarchive/result-type'; +import type { SearchParams } from '../search-params'; +import type { Result } from '@internetarchive/result-type'; import { SearchServiceError, SearchServiceErrorType, } from '../search-service-error'; -import { SearchParamURLGenerator } from '../search-param-url-generator'; +import type { SearchBackendOptionsInterface } from './search-backend-options'; /** - * The DefaultSearchBackend performs a `window.fetch` request to archive.org + * An abstract base class for search backends. */ -export class DefaultSearchBackend implements SearchBackendInterface { - private baseUrl: string; +export abstract class BaseSearchBackend implements SearchBackendInterface { + /** + * The base URL / host this backend should use for its requests. + * Defaults to 'archive.org'. + */ + protected baseUrl: string; - private includeCredentials: boolean; + protected includeCredentials: boolean; - private requestScope?: string; + protected requestScope?: string; - constructor(options?: { - baseUrl?: string; - includeCredentials?: boolean; - scope?: string; - }) { + constructor(options?: SearchBackendOptionsInterface) { this.baseUrl = options?.baseUrl ?? 'archive.org'; if (options?.includeCredentials !== undefined) { @@ -49,34 +49,16 @@ export class DefaultSearchBackend implements SearchBackendInterface { } /** @inheritdoc */ - async performSearch( + abstract performSearch( params: SearchParams - ): Promise> { - const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( - params - ); - const queryAsString = urlSearchParam.toString(); - const url = `https://${this.baseUrl}/advancedsearch.php?${queryAsString}`; - return this.fetchUrl(url); - } - - /** @inheritdoc */ - async fetchMetadata( - identifier: string, - keypath?: string - ): Promise> { - const path = keypath ? `/${keypath}` : ''; - const url = `https://${this.baseUrl}/metadata/${identifier}${path}`; - // the metadata endpoint doesn't current support credentialed requests - // so don't include credentials until that is fixed - return this.fetchUrl(url, { - requestOptions: { - credentials: 'omit', - }, - }); - } + ): Promise>; - private async fetchUrl( + /** + * Fires a request to the URL (with this backend's options applied) and + * asynchronously returns a Result object containing either the raw response + * JSON or a SearchServiceError. + */ + protected async fetchUrl( url: string, options?: { requestOptions?: RequestInit; @@ -107,6 +89,11 @@ export class DefaultSearchBackend implements SearchBackendInterface { // then try json decoding and return a decodingError if it fails try { const json = await response.json(); + + if (json['debugging']) { + this.printDebuggingInfo(json); + } + // the advanced search endpoint doesn't return an HTTP Error 400 // and instead returns an HTTP 200 with an `error` key in the payload const error = json['error']; @@ -141,4 +128,25 @@ export class DefaultSearchBackend implements SearchBackendInterface { const result = { error }; return result; } + + /** + * Logs PPS debugging info to the console if it is present on the response object + */ + private printDebuggingInfo(json: Record) { + const debugInfo = json.debugging?.debugging; // PPS debugging info is doubly-nested, not sure why + const messages = debugInfo?.messages ?? []; + const data = debugInfo?.data ?? {}; + + console.group('Debug messages'); + for (const message of messages) { + console.log(message); + } + console.groupEnd(); + + console.group('Debug data'); + for (const [key, val] of Object.entries(data)) { + console.log(key, val); + } + console.groupEnd(); + } } diff --git a/src/search-backend/fulltext-search-backend.ts b/src/search-backend/fulltext-search-backend.ts new file mode 100644 index 00000000..d76073d1 --- /dev/null +++ b/src/search-backend/fulltext-search-backend.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { SearchParams } from '../search-params'; +import type { Result } from '@internetarchive/result-type'; +import type { SearchServiceError } from '../search-service-error'; +import { SearchParamURLGenerator } from '../search-param-url-generator'; +import { BaseSearchBackend } from './base-search-backend'; +import type { SearchBackendOptionsInterface } from './search-backend-options'; + +/** + * The FulltextSearchBackend performs a `window.fetch` request to archive.org + */ +export class FulltextSearchBackend extends BaseSearchBackend { + private servicePath: string; + + constructor( + options?: SearchBackendOptionsInterface & { + servicePath?: string; + } + ) { + super(options); + this.servicePath = + options?.servicePath ?? '/services/search/beta/page_production'; + } + + /** @inheritdoc */ + async performSearch( + params: SearchParams + ): Promise> { + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const url = `https://${this.baseUrl}${this.servicePath}/?service_backend=fts&${queryAsString}`; + return this.fetchUrl(url); + } +} diff --git a/src/search-backend/metadata-search-backend.ts b/src/search-backend/metadata-search-backend.ts new file mode 100644 index 00000000..5dde6d5f --- /dev/null +++ b/src/search-backend/metadata-search-backend.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { SearchParams } from '../search-params'; +import type { Result } from '@internetarchive/result-type'; +import type { SearchServiceError } from '../search-service-error'; +import { SearchParamURLGenerator } from '../search-param-url-generator'; +import { BaseSearchBackend } from './base-search-backend'; +import type { SearchBackendOptionsInterface } from './search-backend-options'; + +/** + * The MetadataSearchBackend performs a `window.fetch` request to archive.org + */ +export class MetadataSearchBackend extends BaseSearchBackend { + private servicePath: string; + + constructor( + options?: SearchBackendOptionsInterface & { + servicePath?: string; + } + ) { + super(options); + this.servicePath = + options?.servicePath ?? '/services/search/beta/page_production'; + } + + /** @inheritdoc */ + async performSearch( + params: SearchParams + ): Promise> { + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const url = `https://${this.baseUrl}${this.servicePath}/?service_backend=metadata&${queryAsString}`; + return this.fetchUrl(url); + } +} diff --git a/src/search-backend/search-backend-interface.ts b/src/search-backend/search-backend-interface.ts index 3d453610..9d63243c 100644 --- a/src/search-backend/search-backend-interface.ts +++ b/src/search-backend/search-backend-interface.ts @@ -1,14 +1,13 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { Result } from '@internetarchive/result-type'; -import { SearchParams } from '../search-params'; -import { SearchServiceError } from '../search-service-error'; +import type { Result } from '@internetarchive/result-type'; +import type { SearchParams } from '../search-params'; +import type { SearchServiceError } from '../search-service-error'; /** * An interface to provide the network layer to the `SearchService`. * * Objects implementing this interface are responsible for making calls to the Internet Archive - * `advancedsearch` and `metadata` endpoints or otherwise providing a similar reponse in JSON - * format. + * search endpoints or otherwise providing a similar reponse in JSON format. * * This allows for projects like the DWeb project to provide * alternative datasources for a request. @@ -23,15 +22,4 @@ export interface SearchBackendInterface { * @param params */ performSearch(params: SearchParams): Promise>; - - /** - * Fetch metadata for a single item with an optional keypath - * - * @param identifier - * @param keypath - */ - fetchMetadata( - identifier: string, - keypath?: string - ): Promise>; } diff --git a/src/search-backend/search-backend-options.ts b/src/search-backend/search-backend-options.ts new file mode 100644 index 00000000..d913e590 --- /dev/null +++ b/src/search-backend/search-backend-options.ts @@ -0,0 +1,12 @@ +/** + * Options that can be passed to a search backend constructor + */ +export interface SearchBackendOptionsInterface { + /** + * The base URL / host this backend should use for its requests. + * Defaults to 'archive.org'. + */ + baseUrl?: string; + includeCredentials?: boolean; + scope?: string; +} diff --git a/src/search-param-url-generator.ts b/src/search-param-url-generator.ts index dd8a27d7..91e80b20 100644 --- a/src/search-param-url-generator.ts +++ b/src/search-param-url-generator.ts @@ -25,12 +25,16 @@ export class SearchParamURLGenerator { * } * ] * - * @returns string | undefined} + * @returns {string | undefined} * @memberof AggregateSearchParams */ static aggregateSearchParamsAsString( aggregateSearchParams: AggregateSearchParams ): string | undefined { + if (aggregateSearchParams.omit) { + return 'false'; + } + if (aggregateSearchParams.advancedParams) { const params = aggregateSearchParams.advancedParams.map(param => { return { @@ -47,24 +51,31 @@ export class SearchParamURLGenerator { } static sortParamsAsString(sortParams: SortParam): string { - return `${sortParams.field} ${sortParams.direction}`; + return `${sortParams.field}:${sortParams.direction}`; } static generateURLSearchParams(searchParams: SearchParams): URLSearchParams { const params: URLSearchParams = new URLSearchParams(); - params.append('q', searchParams.query); - params.append('output', 'json'); + params.append('user_query', searchParams.query); + + if (searchParams.pageType) { + params.append('page_type', String(searchParams.pageType)); + } + + if (searchParams.pageTarget) { + params.append('page_target', String(searchParams.pageTarget)); + } - if (searchParams.rows) { - params.append('rows', String(searchParams.rows)); + if (searchParams.rows != null) { + params.append('hits_per_page', String(searchParams.rows)); } - if (searchParams.page) { + if (searchParams.page != null) { params.append('page', String(searchParams.page)); } if (searchParams.fields) { - params.append('fl', searchParams.fields.join(',')); + params.append('fields', searchParams.fields.join(',')); } if (searchParams.sort) { @@ -78,10 +89,18 @@ export class SearchParamURLGenerator { if (aggregations) { const aggString = this.aggregateSearchParamsAsString(aggregations); if (aggString) { - params.append('user_aggs', aggString); + params.append('aggregations', aggString); } } + if (searchParams.aggregationsSize != null) { + params.append('aggregations_size', String(searchParams.aggregationsSize)); + } + + if (searchParams.debugging) { + params.append('debugging', 'true'); + } + return params; } } diff --git a/src/search-params.ts b/src/search-params.ts index 3be4f0a4..28ec5caf 100644 --- a/src/search-params.ts +++ b/src/search-params.ts @@ -3,16 +3,48 @@ export interface AggregateSearchParam { size?: number; } +/** + * An object specifying which aggregation types should be returned with + * a search query. + */ export interface AggregateSearchParams { + /** + * An array of objects each specifying both a field name for which + * aggregations should be returned and the number of "buckets" that + * should be returned for it. + * + * Note: this format may not be supported by all backends. Run some + * test queries with advanced aggregation objects before relying on this. + */ advancedParams?: AggregateSearchParam[]; + /** + * An array of field names for which aggregations should be returned. + */ simpleParams?: string[]; + + /** + * A flag to indicate that aggregations should be omitted entirely + * from the response (e.g., to retrieve only search results without + * the added time cost of generating aggregations for them). + * + * Setting this to `true` in a search call will result in a response + * with `undefined` aggregations. + */ + omit?: boolean; } export type SortDirection = 'asc' | 'desc'; export interface SortParam { + /** + * The name of the field to sort on (e.g., 'title'). + */ field: string; + + /** + * Which direction to sort in ('asc' or 'desc'). + */ direction: SortDirection; } @@ -20,20 +52,74 @@ export interface SortParam { * SearchParams provides an encapsulation to all of the search parameters * available for searching. * - * It also provides an `asUrlSearchParams` method for converting the - * parameters to an IA-style query string. ie. it converts the `fields` array - * to `fl=identifier,collection` and `sort` to `sort=date+desc,downloads+asc` + * We use `SearchParamURLGenerator.generateUrlSearchParams` to convert the + * parameters to a PPS-conforming query string -- i.e., it converts the + * `fields` array to `fields=identifier,collection` and `sort` to + * `sort=date:desc,downloads:asc` */ export interface SearchParams { + /** + * The query string to search for. + */ query: string; + /** + * The page type to generate results for (e.g., 'search_results'). + * + * Defaults to 'search_results' in the PPS. Meant to allow different + * backend defaults to be used depending on the needs of certain pages. + */ + pageType?: string; + + /** + * For collection details pages, specifies the name of the collection + * that is to be retrieved (e.g., 'prelinger'). + */ + pageTarget?: string; + + /** + * One or more parameters specifying how the search results should be + * sorted. Each parameter should include the field name and sort direction, + * e.g.: `{ field: 'title', sort: 'desc' }` + */ sort?: SortParam[]; + /** + * The number of results to be retrieved per page. + */ rows?: number; + /** + * The page number to be retrieved (beginning from page 1). + * + * Note that the _size_ of each page is determined by the `rows` parameter. + */ page?: number; + /** + * A list of fields that should be retrieved for each search result. In most + * cases it should be unnecessary to specify the fields parameter as it is + * defaulted by the PPS depending on the page_type. However, it may be useful + * in some cases to restrict which fields are returned to a smaller subset of + * the defaults. + */ fields?: string[]; + /** + * An object specifying which aggregation types should be returned with + * a search query. + */ aggregations?: AggregateSearchParams; + + /** + * The number of buckets that should be returned for each aggregation type. + * This defaults to 6 in the PPS (the number of facets displayed for each + * facet type by default in the sidebar). + */ + aggregationsSize?: number; + + /** + * Whether to include debugging info in the returned PPS response. + */ + debugging?: boolean; } diff --git a/src/search-service-interface.ts b/src/search-service-interface.ts index 7effc6aa..ddad1e35 100644 --- a/src/search-service-interface.ts +++ b/src/search-service-interface.ts @@ -1,65 +1,21 @@ -import { MetadataResponse } from './responses/metadata/metadata-response'; -import { Result } from '@internetarchive/result-type'; -import { SearchResponse } from './responses/search/search-response'; -import { SearchParams } from './search-params'; -import { SearchServiceError } from './search-service-error'; +import type { Result } from '@internetarchive/result-type'; +import type { SearchResponse } from './responses/search-response'; +import type { SearchParams } from './search-params'; +import type { SearchServiceError } from './search-service-error'; +import type { SearchType } from './search-type'; export interface SearchServiceInterface { /** - * Perform a search for given search params + * Perform a search for given search params. * - * @param {SearchParams} params + * @param {SearchParams} params Params object specifying the search query, + * sorting/aggregation options, and other ways to adjust what is returned. + * @param {SearchType} searchType What type of search to perform (e.g., + * metadata or full text) * @returns {Promise>} - * @memberof SearchServiceInterface */ search( - params: SearchParams + params: SearchParams, + searchType?: SearchType ): Promise>; - - /** - * Fetch metadata for a given identifier - * - * @param {string} identifier - * @returns {Promise>} - * @memberof SearchServiceInterface - */ - fetchMetadata( - identifier: string - ): Promise>; - - /** - * Fetch the metadata value for a given identifier and keypath - * - * The response from this request can take any form, object, array, string, etc. - * depending on the query. You can provide return typing in the response by - * specifying the type. Note, there is no automatic type conversion since it can be anything. - * - * For example: - * - * ```ts - * const collection = await searchService.fetchMetadataValue('goody', 'metadata/collection/0'); - * console.debug('collection:', collection); => 'Goody Collection' - * - * const files_count = await searchService.fetchMetadataValue('goody', 'files_count'); - * console.debug('files_count:', files_count); => 12 - * ``` - * - * Keypath examples: - * - * /metadata/:identifier/metadata // returns the entire metadata object - * /metadata/:identifier/server // returns the server for the given identifier - * /metadata/:identifier/files_count - * /metadata/:identifier/files?start=1&count=2 // query for files - * /metadata/:identifier/metadata/collection // all collections - * /metadata/:identifier/metadata/collection/0 // first collection - * /metadata/:identifier/metadata/title - * /metadata/:identifier/files/0/name // first file name - * - * @param identifier - * @param keypath - */ - fetchMetadataValue( - identifier: string, - keypath: string - ): Promise>; } diff --git a/src/search-service.ts b/src/search-service.ts index 11fac818..f9bff041 100644 --- a/src/search-service.ts +++ b/src/search-service.ts @@ -1,79 +1,69 @@ -import { SearchResponse } from './responses/search/search-response'; -import { SearchParams } from './search-params'; -import { MetadataResponse } from './responses/metadata/metadata-response'; -import { DefaultSearchBackend } from './search-backend/default-search-backend'; -import { - SearchServiceError, - SearchServiceErrorType, -} from './search-service-error'; -import { SearchServiceInterface } from './search-service-interface'; -import { SearchBackendInterface } from './search-backend/search-backend-interface'; -import { Result } from '@internetarchive/result-type'; +import { SearchResponse } from './responses/search-response'; +import type { SearchParams } from './search-params'; +import type { SearchServiceError } from './search-service-error'; +import type { SearchServiceInterface } from './search-service-interface'; +import type { Result } from '@internetarchive/result-type'; +import { SearchType } from './search-type'; +import type { SearchBackendOptionsInterface } from './search-backend/search-backend-options'; +import type { SearchBackendInterface } from './search-backend/search-backend-interface'; +import { FulltextSearchBackend } from './search-backend/fulltext-search-backend'; +import { MetadataSearchBackend } from './search-backend/metadata-search-backend'; +import { Memoize } from 'typescript-memoize'; /** * The Search Service is responsible for taking the raw response provided by - * the Search Backend and modeling it as a `SearchResponse` or `MetadataResponse` - * object, depending on the type of response. + * the Search Backend and modeling it as a `SearchResponse` object. */ export class SearchService implements SearchServiceInterface { - public static default: SearchServiceInterface = new SearchService( - new DefaultSearchBackend() - ); + public static default: SearchServiceInterface = new SearchService(); - private searchBackend: SearchBackendInterface; + private backendOptions: SearchBackendOptionsInterface; - constructor(searchBackend: SearchBackendInterface) { - this.searchBackend = searchBackend; + constructor(backendOptions: SearchBackendOptionsInterface = {}) { + this.backendOptions = backendOptions; } /** @inheritdoc */ async search( - params: SearchParams + params: SearchParams, + searchType: SearchType = SearchType.METADATA ): Promise> { - const rawResponse = await this.searchBackend.performSearch(params); - if (rawResponse.error) { - return rawResponse; - } - - const modeledResponse = new SearchResponse(rawResponse.success); - return { success: modeledResponse }; - } + const searchBackend = SearchService.getBackendForSearchType( + searchType, + this.backendOptions + ); - /** @inheritdoc */ - async fetchMetadata( - identifier: string - ): Promise> { - const rawResponse = await this.searchBackend.fetchMetadata(identifier); + const rawResponse = await searchBackend.performSearch(params); if (rawResponse.error) { return rawResponse; } - if (rawResponse.success?.metadata === undefined) { - return { - error: new SearchServiceError(SearchServiceErrorType.itemNotFound), - }; - } - - const modeledResponse = new MetadataResponse(rawResponse.success); + const modeledResponse = new SearchResponse(rawResponse.success); return { success: modeledResponse }; } - /** @inheritdoc */ - async fetchMetadataValue( - identifier: string, - keypath: string - ): Promise> { - const result = await this.searchBackend.fetchMetadata(identifier, keypath); - if (result.error) { - return result; + /** + * Retrieve a search backend that can handle the given type of search. + * @param type The type of search that the backend needs to handle. + * @param options Options to pass to the search backend. + */ + @Memoize((type: SearchType, options: SearchBackendOptionsInterface = {}) => { + // We can memoize backends based on their params, to avoid constructing redundant backends + const { includeCredentials = '', scope = '', baseUrl = '' } = options; + return `${type};${includeCredentials};${scope};${baseUrl}`; + }) + static getBackendForSearchType( + type: SearchType, + options: SearchBackendOptionsInterface = {} + ): SearchBackendInterface { + switch (type) { + //case SearchType.TV: // Will eventually have its own service backend + //case SearchType.RADIO: // Will eventually have its own service backend + case SearchType.FULLTEXT: + return new FulltextSearchBackend(options); + case SearchType.METADATA: + default: + return new MetadataSearchBackend(options); } - - if (result.success?.result === undefined) { - return { - error: new SearchServiceError(SearchServiceErrorType.itemNotFound), - }; - } - - return { success: result.success.result }; } } diff --git a/src/search-type.ts b/src/search-type.ts new file mode 100644 index 00000000..4d245a56 --- /dev/null +++ b/src/search-type.ts @@ -0,0 +1,10 @@ +/** + * An enum specifying the different types of search that can be conducted + * through the SearchService. + */ +export enum SearchType { + METADATA, + FULLTEXT, + //TV, // not yet available + //RADIO, // not yet available +} diff --git a/test/default-search-backend.test.ts b/test/default-search-backend.test.ts deleted file mode 100644 index 4f87ab41..00000000 --- a/test/default-search-backend.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { expect } from '@open-wc/testing'; - -import { SearchParams } from '../src/search-params'; - -import { SearchServiceErrorType } from '../src/search-service-error'; -import { DefaultSearchBackend } from '../src/search-backend/default-search-backend'; - -describe('DefaultSearchBackend', () => { - it('can fetch metadata', async () => { - const fetchBackup = window.fetch; - window.fetch = (): Promise => { - return new Promise(resolve => { - const response = new Response('{ "foo": "bar" }'); - resolve(response); - }); - }; - - const backend = new DefaultSearchBackend(); - const result = await backend.fetchMetadata('foo'); - expect(result.success?.foo).to.equal('bar'); - window.fetch = fetchBackup; - }); - - it('can perform a search', async () => { - const fetchBackup = window.fetch; - window.fetch = (): Promise => { - return new Promise(resolve => { - const response = new Response('{ "foo": "bar" }'); - resolve(response); - }); - }; - - const backend = new DefaultSearchBackend(); - const params = { query: 'foo' }; - const result = await backend.performSearch(params); - expect(result.success?.foo).to.equal('bar'); - window.fetch = fetchBackup; - }); - - it('returns a networkError if theres a problem fetching using String type', async () => { - const fetchBackup = window.fetch; - window.fetch = (): Promise => { - throw 'network error'; - }; - - const backend = new DefaultSearchBackend(); - const result = await backend.fetchMetadata('foo'); - expect(result.error?.type).to.equal(SearchServiceErrorType.networkError); - expect(result.error?.message).to.equal('network error'); - window.fetch = fetchBackup; - }); - - it('returns a networkError if theres a problem fetching using Error type', async () => { - const fetchBackup = window.fetch; - window.fetch = (): Promise => { - throw new Error('network error'); - }; - - const backend = new DefaultSearchBackend(); - const result = await backend.fetchMetadata('foo'); - expect(result.error?.type).to.equal(SearchServiceErrorType.networkError); - expect(result.error?.message).to.equal('network error'); - window.fetch = fetchBackup; - }); - - it('returns a decodingError if theres a problem decoding the json', async () => { - const fetchBackup = window.fetch; - window.fetch = (): Promise => { - const response = new Response('boop'); - return new Promise(resolve => resolve(response)); - }; - - const backend = new DefaultSearchBackend(); - const result = await backend.fetchMetadata('foo'); - expect(result.error?.type).to.equal(SearchServiceErrorType.decodingError); - window.fetch = fetchBackup; - }); - - it('appends the scope if provided', async () => { - const fetchBackup = window.fetch; - let urlCalled = ''; - window.fetch = (url: RequestInfo): Promise => { - urlCalled = url.toString(); - const response = new Response('boop'); - return new Promise(resolve => resolve(response)); - }; - - const backend = new DefaultSearchBackend({ - scope: 'foo', - }); - const result = await backend.fetchMetadata('foo'); - expect(urlCalled.includes('scope=foo')).to.be.true; - window.fetch = fetchBackup; - }); - - it('includes credentials for search endpoint if requested', async () => { - const fetchBackup = window.fetch; - let urlCalled: RequestInfo; - let urlConfig: RequestInit | undefined; - window.fetch = ( - url: RequestInfo, - config?: RequestInit - ): Promise => { - urlCalled = url; - urlConfig = config; - const response = new Response('boop'); - return new Promise(resolve => resolve(response)); - }; - - const backend = new DefaultSearchBackend({ - scope: 'foo', - includeCredentials: true, - }); - await backend.performSearch({ query: 'foo' }); - expect(urlConfig?.credentials).to.equal('include'); - window.fetch = fetchBackup; - }); - - it('does not credentials for metadata endpoint', async () => { - const fetchBackup = window.fetch; - let urlCalled: RequestInfo; - let urlConfig: RequestInit | undefined; - window.fetch = ( - url: RequestInfo, - config?: RequestInit - ): Promise => { - urlCalled = url; - urlConfig = config; - const response = new Response('boop'); - return new Promise(resolve => resolve(response)); - }; - - const backend = new DefaultSearchBackend({ - scope: 'foo', - includeCredentials: true, - }); - await backend.fetchMetadata('foo'); - expect(urlConfig?.credentials).to.equal('omit'); - window.fetch = fetchBackup; - }); -}); diff --git a/test/mock-response-generator.ts b/test/mock-response-generator.ts index 09a92ee6..5186162f 100644 --- a/test/mock-response-generator.ts +++ b/test/mock-response-generator.ts @@ -1,102 +1,114 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ - -import { SearchResponse } from '../src/responses/search/search-response'; import { SearchParams } from '../src/search-params'; -import { Metadata } from '../src/models/metadata'; +import { ItemHit } from '../src/models/hit-types/item-hit'; +import { TextHit } from '../src/models/hit-types/text-hit'; export class MockResponseGenerator { - generateMockSearchResponse(params: SearchParams): SearchResponse { - const fieldsAsString = params.fields?.join(','); - - const metadata1 = new Metadata({ identifier: 'foo' }); - const metadata2 = new Metadata({ identifier: 'bar' }); + generateMockMetadataSearchResponse( + params: SearchParams + ): Record { + const metadata1 = new ItemHit({ identifier: 'foo' }); + const metadata2 = new ItemHit({ identifier: 'bar' }); return { rawResponse: { foo: 'bar', }, - responseHeader: { - status: 0, - QTime: 1459, - params: { - query: params.query, - qin: params.query, - fields: fieldsAsString ?? '', - wt: 'json', - sort: params.sort?.reduce((prev, current, index) => { - const isFirst = index === 0; - const commaPrefix = isFirst ? '' : ', '; - return `${prev}${commaPrefix}${current.field} ${current.direction}`; - }, ''), - rows: params.rows ? `${params.rows}` : '', - start: params.page ?? 0, + request: { + client_parameters: { + client: 'page_production_service_endpoint', + user_query: params.query, + page: params.page ?? 0, + hits_per_page: params.rows ?? 0, + fields: params.fields ?? [], + }, + finalized_parameters: { + client: 'page_production_service_endpoint', + user_query: params.query, + service_backend: 'metadata', + page: params.page ?? 0, + page_type: 'search_results', + hits_per_page: 50, + fields: ['_tile_'].concat(params.fields ?? []), + sort: ['_score'], + aggregations: [ + 'mediatype', + 'year', + 'subject', + 'collection', + 'creator', + 'language', + ], + aggregations_size: 6, }, }, + responseHeader: { + succeeded: true, + query_time: 3133, + }, response: { - numFound: 12345, - start: 0, - docs: [metadata1, metadata2], + totalResults: 12345, + returnedCount: 2, + results: [metadata1, metadata2], + schema: { + hit_type: 'item', + field_properties: {}, + }, }, }; } - generateMockMetadataResponse(identifier: string): any { + generateMockFulltextSearchResponse( + params: SearchParams + ): Record { + const text1 = new TextHit({ identifier: 'foo' }); + const text2 = new TextHit({ identifier: 'bar' }); + return { - created: 1586477049, - d1: 'ia600201.us.archive.org', - d2: 'ia800201.us.archive.org', - dir: '/27/items/rss-383924main_TWAN_09_04_09', - files: [ - { - name: 'foo.jpg', - source: 'derivative', - format: 'Thumbnail', - original: 'foo.mp4', - md5: '48067b43a547d3e90cb433a04ba84d5d', - mtime: '1256675427', - size: '1135', - crc32: '40870038', - sha1: '6445c2a6f51c8314c1872ec1f7daf33c5cfabd06', + rawResponse: { + foo: 'bar', + }, + request: { + client_parameters: { + client: 'page_production_service_endpoint', + user_query: params.query, + page: params.page ?? 0, + hits_per_page: params.rows ?? 0, + fields: params.fields ?? [], + }, + finalized_parameters: { + client: 'page_production_service_endpoint', + user_query: params.query, + service_backend: 'fts', + page: params.page ?? 0, + page_type: 'search_results', + hits_per_page: 50, + fields: ['_tile_'].concat(params.fields ?? []), + sort: ['_score'], + aggregations: [ + 'mediatype', + 'year', + 'subject', + 'collection', + 'creator', + 'language', + ], + aggregations_size: 6, }, - { - name: 'bar.jpg', - source: 'derivative', - format: 'Thumbnail', - original: 'bar.mp4', - md5: '5bf69912d7b796fe309cde32e61230bc', - mtime: '1256675429', - size: '6329', - crc32: 'ba5f361a', - sha1: 'cddab0e2daab29978e5efdeff735240a44aa7c80', + }, + responseHeader: { + succeeded: true, + query_time: 3133, + }, + response: { + totalResults: 12345, + returnedCount: 2, + results: [text1, text2], + schema: { + hit_type: 'text', + field_properties: {}, }, - ], - files_count: 2, - item_last_updated: 1463797130, - item_size: 99872691, - metadata: { - feed_id: - '/0/rss_feeds/NASACast_Video/nasacast_video:sts128_landing/NASAcast_vodcast.rss', - mediatype: 'movies', - title: "NASA TV's This Week @NASA, September 4", - description: "NASA TV's This Week @NASA, September 4", - creator: 'NASA', - source: 'http://www.nasa.gov/multimedia/podcasting/twan_09_04_09.html', - date: '9/4/2009', - year: '2009', - rights: 'Public Domain', - language: 'en-us', - updater: 'BonnieReal', - updatedate: '2009-10-27 20:23:01', - identifier, - uploader: 'bonnie@archive.org', - addeddate: '2009-10-27 20:25:55', - publicdate: '2009-10-27 20:41:27', - collection: ['nasa', 'nasacastvideo'], - backup_location: 'ia903604_7', }, - server: 'ia800201.us.archive.org', - uniq: 162444403, - workable_servers: ['ia800201.us.archive.org', 'ia600201.us.archive.org'], }; } } diff --git a/test/models/file.test.ts b/test/models/file.test.ts deleted file mode 100644 index 6ca7b91c..00000000 --- a/test/models/file.test.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { expect } from '@open-wc/testing'; - -import { File } from '../../src/models/file'; - -describe('File', () => { - it('can be instantiated with an object', async () => { - const file = new File({ name: 'foo.jpg' }); - expect(file.name).to.equal('foo.jpg'); - }); -}); diff --git a/test/models/hit-types/item-hit.test.ts b/test/models/hit-types/item-hit.test.ts new file mode 100644 index 00000000..e4d34b29 --- /dev/null +++ b/test/models/hit-types/item-hit.test.ts @@ -0,0 +1,144 @@ +import { DateParser } from '@internetarchive/field-parsers'; +import { expect } from '@open-wc/testing'; +import { ItemHit } from '../../../src/models/hit-types/item-hit'; +import { DateField } from '../../../src/models/metadata-fields/field-types/date'; + +describe('ItemHit', () => { + it('constructs basic item hit', () => { + const hit = new ItemHit({ fields: {} }); + expect(hit.rawMetadata).to.deep.equal({ fields: {} }); + + // Small selection of fields + expect(hit.creator).to.be.undefined; + expect(hit.date).to.be.undefined; + expect(hit.description).to.be.undefined; + expect(hit.subject).to.be.undefined; + expect(hit.title).to.be.undefined; + }); + + it('handles incomplete data without throwing', () => { + const hit = new ItemHit({}); + expect(hit.creator).to.be.undefined; + expect(hit.date).to.be.undefined; + expect(hit.description).to.be.undefined; + expect(hit.subject).to.be.undefined; + expect(hit.title).to.be.undefined; + }); + + it('constructs item hit with partial fields', () => { + const json = { + fields: { + identifier: 'foo', + title: 'foo-title', + description: 'foo-description', + subject: ['foo-subject1', 'foo-subject2'], + creator: ['foo-creator'], + collection: ['foo-collection'], + date: '2011-07-20T00:00:00Z', + year: 2011, + mediatype: 'movies', + item_size: 123456, + files_count: 5, + downloads: 123, + week: 2, + month: 15, + indexflag: ['index', 'nonoindex'], + lending___available_to_borrow: false, + lending___available_to_browse: false, + lending___available_to_waitlist: false, + }, + highlight: null, + _score: 1, + }; + + const hit = new ItemHit(json); + expect(hit.rawMetadata).to.deep.equal(json); + expect(hit.identifier).to.equal(json.fields.identifier); + + // Ensure all existing fields are present + for (const key of Object.keys(json.fields)) { + if (key === 'identifier') continue; + const fieldName = key as Exclude; + + if (Array.isArray(json.fields[fieldName])) { + expect(hit[fieldName]?.values).to.deep.equal(json.fields[fieldName]); + } else if (hit[fieldName] instanceof DateField) { + expect(hit[fieldName]?.value).to.deep.equal( + DateParser.shared.parseValue(json.fields[fieldName].toString()) + ); + } else { + expect(hit[fieldName]?.value).to.equal(json.fields[fieldName]); + } + } + + expect(hit.collection_files_count?.value).to.be.undefined; + expect(hit.collection_size?.value).to.be.undefined; + expect(hit.genre?.value).to.be.undefined; + expect(hit.language?.value).to.be.undefined; + expect(hit.lending___status?.value).to.be.undefined; + expect(hit.licenseurl?.value).to.be.undefined; + expect(hit.noindex?.value).to.be.undefined; + expect(hit.num_favorites?.value).to.be.undefined; + expect(hit.num_reviews?.value).to.be.undefined; + expect(hit.type?.value).to.be.undefined; + expect(hit.volume?.value).to.be.undefined; + }); + + it('constructs item hit with all fields', () => { + const json = { + fields: { + identifier: 'foo', + title: 'foo-title', + description: 'foo-description', + subject: ['foo-subject1', 'foo-subject2'], + creator: ['foo-creator'], + collection: ['foo-collection'], + date: '2011-07-20T00:00:00Z', + year: 2011, + mediatype: 'movies', + item_size: 123456, + files_count: 5, + downloads: 123, + week: 2, + month: 15, + indexflag: ['index', 'nonoindex'], + lending___available_to_borrow: false, + lending___available_to_browse: false, + lending___available_to_waitlist: false, + lending___status: 'foo-status', + collection_files_count: 124, + collection_size: 125, + genre: 'foo-genre', + language: 'foo-language', + licenseurl: 'foo-license', + noindex: true, + num_favorites: 126, + num_reviews: 127, + type: 'foo-type', + volume: 'foo-volume', + }, + highlight: null, + _score: 1, + }; + + const hit = new ItemHit(json); + expect(hit.rawMetadata).to.deep.equal(json); + expect(hit.identifier).to.equal(json.fields.identifier); + + // Ensure all existing fields are present + for (const key of Object.keys(json.fields)) { + if (key === 'identifier') continue; + const fieldName = key as Exclude; + + if (Array.isArray(json.fields[fieldName])) { + expect(hit[fieldName]?.values).to.deep.equal(json.fields[fieldName]); + } else if (hit[fieldName] instanceof DateField) { + expect(hit[fieldName]?.value).to.deep.equal( + DateParser.shared.parseValue(json.fields[fieldName].toString()) + ); + } else { + expect(hit[fieldName]?.value).to.equal(json.fields[fieldName]); + } + } + }); +}); diff --git a/test/models/hit-types/text-hit.test.ts b/test/models/hit-types/text-hit.test.ts new file mode 100644 index 00000000..b1492349 --- /dev/null +++ b/test/models/hit-types/text-hit.test.ts @@ -0,0 +1,73 @@ +import { DateParser } from '@internetarchive/field-parsers'; +import { expect } from '@open-wc/testing'; +import { TextHit } from '../../../src/models/hit-types/text-hit'; +import { DateField } from '../../../src/models/metadata-fields/field-types/date'; + +describe('TextHit', () => { + it('constructs basic text hit', () => { + const hit = new TextHit({ fields: {} }); + expect(hit.rawMetadata).to.deep.equal({ fields: {} }); + expect(hit.creator).to.be.undefined; + }); + + it('handles incomplete data without throwing', () => { + const hit = new TextHit({}); + expect(hit.creator).to.be.undefined; + expect(hit.date).to.be.undefined; + expect(hit.description).to.be.undefined; + expect(hit.subject).to.be.undefined; + expect(hit.title).to.be.undefined; + }); + + it('constructs text hit with all fields', () => { + const json = { + fields: { + identifier: 'foo', + mediatype: 'texts', + filename: 'foo-file.txt', + file_basename: 'foo-file', + file_creation_mtime: 1161893250, + updated_on: '2022-04-06T08:34:38Z', + created_on: '2016-10-09T16:51:05Z', + page_num: 462, + title: 'foo-title', + creator: ['foo-creator'], + subject: ['foo-subject1', 'foo-subject2'], + date: '1904-01-01T00:00:00Z', + publicdate: '2006-10-11T08:19:20Z', + downloads: 1234, + collection: ['foo-collection'], + year: 2011, + result_in_subfile: false, + description: 'foo-description', + __href__: '/details/foo?q=bar', + }, + highlight: { + text: ['foo {{{bar}}} baz'], + }, + _score: 1, + }; + + const hit = new TextHit(json); + expect(hit.rawMetadata).to.deep.equal(json); + expect(hit.identifier).to.equal(json.fields.identifier); + + // Ensure all existing fields are present + for (const key of Object.keys(json.fields)) { + if (key === 'identifier') continue; + const fieldName = key as Exclude; + + if (Array.isArray(json.fields[fieldName])) { + expect(hit[fieldName]?.values).to.deep.equal(json.fields[fieldName]); + } else if (hit[fieldName] instanceof DateField) { + expect(hit[fieldName]?.value).to.deep.equal( + DateParser.shared.parseValue(json.fields[fieldName].toString()) + ); + } else { + expect(hit[fieldName]?.value).to.equal(json.fields[fieldName]); + } + } + + expect(hit.highlight?.values).to.deep.equal(json.highlight.text); + }); +}); diff --git a/test/models/metadata-fields/metadata-field.test.ts b/test/models/metadata-fields/metadata-field.test.ts index 834b00fb..c1ba3f14 100644 --- a/test/models/metadata-fields/metadata-field.test.ts +++ b/test/models/metadata-fields/metadata-field.test.ts @@ -1,7 +1,4 @@ -import { - FieldParserInterface, - FieldParserRawValue, -} from '@internetarchive/field-parsers'; +import { FieldParserInterface } from '@internetarchive/field-parsers'; import { expect } from '@open-wc/testing'; import { MetadataField } from '../../../src/models/metadata-fields/metadata-field'; @@ -88,7 +85,7 @@ describe('Metadata Field', () => { it('does not add value to values array if parsed value is undefined', () => { class MockFloatParser implements FieldParserInterface { - parseValue(rawValue: FieldParserRawValue): number | undefined { + parseValue(): number | undefined { return undefined; } } diff --git a/test/responses/search-response-details.test.ts b/test/responses/search-response-details.test.ts new file mode 100644 index 00000000..4867c1c7 --- /dev/null +++ b/test/responses/search-response-details.test.ts @@ -0,0 +1,79 @@ +import { expect } from '@open-wc/testing'; +import { HitType } from '../../src/models/hit-types/hit'; +import { ItemHit } from '../../src/models/hit-types/item-hit'; +import { TextHit } from '../../src/models/hit-types/text-hit'; +import { SearchResponseDetails } from '../../src/responses/search-response-details'; + +describe('SearchResponseDetails', () => { + it('constructs item hits', () => { + const responseBody = { + hits: { + total: 2, + returned: 2, + hits: [ + { + fields: { + identifier: 'foo', + mediatype: 'texts', + }, + }, + { + fields: { + identifier: 'bar', + collection: ['baz'], + }, + }, + ], + }, + }; + + const responseSchema = { + hit_type: 'item' as HitType, + field_properties: {}, + }; + + const details = new SearchResponseDetails(responseBody, responseSchema); + expect(details.results[0]).to.be.instanceOf(ItemHit); + expect(details.results[0].identifier).to.equal('foo'); + expect(details.results[0].mediatype?.value).to.equal('texts'); + expect(details.results[0].creator?.value).to.be.undefined; + expect(details.results[1].identifier).to.equal('bar'); + expect(details.results[1].collection?.value).to.equal('baz'); + }); + + it('constructs text hits', () => { + const responseBody = { + hits: { + total: 2, + returned: 2, + hits: [ + { + fields: { + identifier: 'foo', + mediatype: 'texts', + }, + }, + { + fields: { + identifier: 'bar', + collection: ['baz'], + }, + }, + ], + }, + }; + + const responseSchema = { + hit_type: 'text' as HitType, + field_properties: {}, + }; + + const details = new SearchResponseDetails(responseBody, responseSchema); + expect(details.results[0]).to.be.instanceOf(TextHit); + expect(details.results[0].identifier).to.equal('foo'); + expect(details.results[0].mediatype?.value).to.equal('texts'); + expect(details.results[0].creator?.value).to.be.undefined; + expect(details.results[1].identifier).to.equal('bar'); + expect(details.results[1].collection?.value).to.equal('baz'); + }); +}); diff --git a/test/search-backend/fulltext-search-backend.test.ts b/test/search-backend/fulltext-search-backend.test.ts new file mode 100644 index 00000000..4925f974 --- /dev/null +++ b/test/search-backend/fulltext-search-backend.test.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { expect } from '@open-wc/testing'; +import Sinon from 'sinon'; +import { FulltextSearchBackend } from '../../src/search-backend/fulltext-search-backend'; +import { + SearchServiceError, + SearchServiceErrorType, +} from '../../src/search-service-error'; + +describe('FulltextSearchBackend', () => { + describe('basic fetch behavior', () => { + let fetchBackup: typeof window.fetch; + let urlCalled: RequestInfo | URL; + let urlConfig: RequestInit | undefined; + + beforeEach(() => { + fetchBackup = window.fetch; + urlCalled = ''; + urlConfig = undefined; + + window.fetch = ( + url: RequestInfo | URL, + config?: RequestInit + ): Promise => { + urlCalled = url; + urlConfig = config; + const response = new Response('{ "foo": "bar" }'); + return new Promise(resolve => resolve(response)); + }; + }); + + afterEach(() => { + window.fetch = fetchBackup; + }); + + it('can perform a search', async () => { + const backend = new FulltextSearchBackend(); + const params = { query: 'foo' }; + const result = await backend.performSearch(params); + + expect(result.success?.foo).to.equal('bar'); + }); + + it('sets the fts service backend', async () => { + const backend = new FulltextSearchBackend(); + await backend.performSearch({ query: 'foo' }); + + expect( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + new URL(urlCalled!.toString()).searchParams.get('service_backend') + ).to.equal('fts'); + }); + + it('uses the provided service path', async () => { + const backend = new FulltextSearchBackend({ + baseUrl: 'foo.bar', + servicePath: '/baz', + }); + await backend.performSearch({ query: 'boop' }); + + expect(urlCalled!.toString()).to.satisfy((url: string) => + url.startsWith('https://foo.bar/baz') + ); + }); + + it('includes credentials for search endpoint if requested', async () => { + const backend = new FulltextSearchBackend({ + scope: 'foo', + includeCredentials: true, + }); + await backend.performSearch({ query: 'foo' }); + + expect(urlConfig?.credentials).to.equal('include'); + }); + }); + + it('returns a network error result upon fetch errors', async () => { + const fetchBackup = window.fetch; + window.fetch = async () => { + throw new Error(); + }; + + const backend = new FulltextSearchBackend(); + const response = await backend.performSearch({ query: 'foo' }); + expect(response).to.exist; // No error thrown + expect(response.error).to.be.instanceof(SearchServiceError); + expect(response.error?.type).to.equal(SearchServiceErrorType.networkError); + + window.fetch = fetchBackup; + }); + + it('outputs debugging information if present', async () => { + const fetchBackup = window.fetch; + let urlCalled: RequestInfo | URL = ''; + window.fetch = async (url: RequestInfo | URL) => { + urlCalled = url; + return new Response( + JSON.stringify({ + debugging: { + debugging: { + messages: ['boop'], + data: { + bar: 'baz', + }, + }, + }, + }) + ); + }; + + const logBackup = console.log; + const logSpy = Sinon.spy(); + console.log = logSpy; + + const backend = new FulltextSearchBackend(); + await backend.performSearch({ query: 'foo', debugging: true }); + expect(urlCalled).to.include('debugging=true'); + expect(logSpy.calledWithExactly('boop')).to.be.true; + expect(logSpy.calledWithExactly('bar', 'baz')).to.be.true; + + window.fetch = fetchBackup; + console.log = logBackup; + }); + + it('handles incomplete debugging information gracefully', async () => { + const fetchBackup = window.fetch; + window.fetch = async () => { + return new Response( + JSON.stringify({ + debugging: {}, + }) + ); + }; + + const logBackup = console.log; + const logSpy = Sinon.spy(); + console.log = logSpy; + + const backend = new FulltextSearchBackend(); + const response = await backend.performSearch({ + query: 'foo', + debugging: true, + }); + expect(response).to.exist; // No error thrown + + window.fetch = fetchBackup; + console.log = logBackup; + }); +}); diff --git a/test/search-backend/metadata-search-backend.test.ts b/test/search-backend/metadata-search-backend.test.ts new file mode 100644 index 00000000..b0ff7496 --- /dev/null +++ b/test/search-backend/metadata-search-backend.test.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { expect } from '@open-wc/testing'; +import Sinon from 'sinon'; +import { MetadataSearchBackend } from '../../src/search-backend/metadata-search-backend'; +import { + SearchServiceError, + SearchServiceErrorType, +} from '../../src/search-service-error'; + +describe('MetadataSearchBackend', () => { + describe('basic fetch behavior', () => { + let fetchBackup: typeof window.fetch; + let urlCalled: RequestInfo | URL; + let urlConfig: RequestInit | undefined; + + beforeEach(() => { + fetchBackup = window.fetch; + urlCalled = ''; + urlConfig = undefined; + + window.fetch = ( + url: RequestInfo | URL, + config?: RequestInit + ): Promise => { + urlCalled = url; + urlConfig = config; + const response = new Response('{ "foo": "bar" }'); + return new Promise(resolve => resolve(response)); + }; + }); + + afterEach(() => { + window.fetch = fetchBackup; + }); + + it('can perform a search', async () => { + const backend = new MetadataSearchBackend(); + const params = { query: 'foo' }; + const result = await backend.performSearch(params); + + expect(result.success?.foo).to.equal('bar'); + }); + + it('sets the metadata service backend', async () => { + const backend = new MetadataSearchBackend(); + await backend.performSearch({ query: 'foo' }); + + expect( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + new URL(urlCalled!.toString()).searchParams.get('service_backend') + ).to.equal('metadata'); + }); + + it('uses the provided service path', async () => { + const backend = new MetadataSearchBackend({ + baseUrl: 'foo.bar', + servicePath: '/baz', + }); + await backend.performSearch({ query: 'boop' }); + + expect(urlCalled!.toString()).to.satisfy((url: string) => + url.startsWith('https://foo.bar/baz') + ); + }); + + it('includes credentials for search endpoint if requested', async () => { + const backend = new MetadataSearchBackend({ + scope: 'foo', + includeCredentials: true, + }); + await backend.performSearch({ query: 'foo' }); + + expect(urlConfig?.credentials).to.equal('include'); + }); + }); + + it('returns a network error result upon fetch errors', async () => { + const fetchBackup = window.fetch; + window.fetch = async () => { + throw new Error(); + }; + + const backend = new MetadataSearchBackend(); + const response = await backend.performSearch({ query: 'foo' }); + expect(response).to.exist; // No error thrown + expect(response.error).to.be.instanceof(SearchServiceError); + expect(response.error?.type).to.equal(SearchServiceErrorType.networkError); + + window.fetch = fetchBackup; + }); + + it('outputs debugging information if present', async () => { + const fetchBackup = window.fetch; + let urlCalled: RequestInfo | URL = ''; + window.fetch = async (url: RequestInfo | URL) => { + urlCalled = url; + return new Response( + JSON.stringify({ + debugging: { + debugging: { + messages: ['boop'], + data: { + bar: 'baz', + }, + }, + }, + }) + ); + }; + + const logBackup = console.log; + const logSpy = Sinon.spy(); + console.log = logSpy; + + const backend = new MetadataSearchBackend(); + await backend.performSearch({ query: 'foo', debugging: true }); + expect(urlCalled).to.include('debugging=true'); + expect(logSpy.calledWithExactly('boop')).to.be.true; + expect(logSpy.calledWithExactly('bar', 'baz')).to.be.true; + + window.fetch = fetchBackup; + console.log = logBackup; + }); + + it('handles incomplete debugging information gracefully', async () => { + const fetchBackup = window.fetch; + window.fetch = async () => { + return new Response( + JSON.stringify({ + debugging: {}, + }) + ); + }; + + const logBackup = console.log; + const logSpy = Sinon.spy(); + console.log = logSpy; + + const backend = new MetadataSearchBackend(); + const response = await backend.performSearch({ + query: 'foo', + debugging: true, + }); + expect(response).to.exist; // No error thrown + + window.fetch = fetchBackup; + console.log = logBackup; + }); +}); diff --git a/test/search-params.test.ts b/test/search-params.test.ts index 161e9d6f..4ecc8857 100644 --- a/test/search-params.test.ts +++ b/test/search-params.test.ts @@ -27,7 +27,7 @@ describe('SearchParams', () => { params ); const queryAsString = urlSearchParam.toString(); - const expected = 'q=title%3Afoo+AND+collection%3Abar&output=json'; + const expected = 'user_query=title%3Afoo+AND+collection%3Abar'; expect(queryAsString).to.equal(expected); }); @@ -43,7 +43,7 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&fl=identifier%2Cfoo%2Cbar'; + 'user_query=title%3Afoo+AND+collection%3Abar&fields=identifier%2Cfoo%2Cbar'; expect(queryAsString).to.equal(expected); }); @@ -63,7 +63,7 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&rows=53&page=27&fl=identifier%2Cfoo%2Cbar&sort=downloads+desc'; + 'user_query=title%3Afoo+AND+collection%3Abar&hits_per_page=53&page=27&fields=identifier%2Cfoo%2Cbar&sort=downloads%3Adesc'; expect(queryAsString).to.equal(expected); }); @@ -84,7 +84,55 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&rows=53&page=27&sort=downloads+desc%2Cfoo+asc'; + 'user_query=title%3Afoo+AND+collection%3Abar&hits_per_page=53&page=27&sort=downloads%3Adesc%2Cfoo%3Aasc'; + expect(queryAsString).to.equal(expected); + }); + + it('properly generates a URLSearchParam with a page_type', async () => { + const query = 'title:foo AND collection:bar'; + const params = { + query, + pageType: 'foo', + }; + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const expected = + 'user_query=title%3Afoo+AND+collection%3Abar&page_type=foo'; + expect(queryAsString).to.equal(expected); + }); + + it('properly generates a URLSearchParam with a page_type and page_target', async () => { + const query = 'title:foo AND collection:bar'; + const params = { + query, + pageType: 'foo', + pageTarget: 'bar', + }; + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const expected = + 'user_query=title%3Afoo+AND+collection%3Abar&page_type=foo&page_target=bar'; + expect(queryAsString).to.equal(expected); + }); + + it('properly generates a URLSearchParam with aggregations omitted', async () => { + const query = 'title:foo AND collection:bar'; + const params = { + query, + aggregations: { + omit: true, + }, + }; + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const expected = + 'user_query=title%3Afoo+AND+collection%3Abar&aggregations=false'; expect(queryAsString).to.equal(expected); }); @@ -111,7 +159,7 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&user_aggs=%5B%7B%22terms%22%3A%7B%22field%22%3A%22foo%22%2C%22size%22%3A10%7D%7D%2C%7B%22terms%22%3A%7B%22field%22%3A%22bar%22%2C%22size%22%3A7%7D%7D%5D'; + 'user_query=title%3Afoo+AND+collection%3Abar&aggregations=%5B%7B%22terms%22%3A%7B%22field%22%3A%22foo%22%2C%22size%22%3A10%7D%7D%2C%7B%22terms%22%3A%7B%22field%22%3A%22bar%22%2C%22size%22%3A7%7D%7D%5D'; expect(queryAsString).to.equal(expected); }); @@ -129,7 +177,26 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&user_aggs=year%2Ccollection%2Csubject'; + 'user_query=title%3Afoo+AND+collection%3Abar&aggregations=year%2Ccollection%2Csubject'; + expect(queryAsString).to.equal(expected); + }); + + it('properly generates a URLSearchParam with an aggregations_size param', async () => { + const query = 'title:foo AND collection:bar'; + const aggregations = { + simpleParams: ['year', 'collection', 'subject'], + }; + const params = { + query, + aggregations, + aggregationsSize: 3, + }; + const urlSearchParam = SearchParamURLGenerator.generateURLSearchParams( + params + ); + const queryAsString = urlSearchParam.toString(); + const expected = + 'user_query=title%3Afoo+AND+collection%3Abar&aggregations=year%2Ccollection%2Csubject&aggregations_size=3'; expect(queryAsString).to.equal(expected); }); @@ -157,7 +224,7 @@ describe('SearchParams', () => { ); const queryAsString = urlSearchParam.toString(); const expected = - 'q=title%3Afoo+AND+collection%3Abar&output=json&user_aggs=%5B%7B%22terms%22%3A%7B%22field%22%3A%22foo%22%2C%22size%22%3A10%7D%7D%2C%7B%22terms%22%3A%7B%22field%22%3A%22bar%22%2C%22size%22%3A7%7D%7D%5D'; + 'user_query=title%3Afoo+AND+collection%3Abar&aggregations=%5B%7B%22terms%22%3A%7B%22field%22%3A%22foo%22%2C%22size%22%3A10%7D%7D%2C%7B%22terms%22%3A%7B%22field%22%3A%22bar%22%2C%22size%22%3A7%7D%7D%5D'; expect(queryAsString).to.equal(expected); }); }); diff --git a/test/search-service.test.ts b/test/search-service.test.ts index 41c78de9..3c3b5e5e 100644 --- a/test/search-service.test.ts +++ b/test/search-service.test.ts @@ -1,159 +1,54 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-explicit-any */ import { expect } from '@open-wc/testing'; - +import sinon from 'sinon'; import { SearchService } from '../src/search-service'; import { SearchParams } from '../src/search-params'; import { MockResponseGenerator } from './mock-response-generator'; -import { SearchResponse } from '../src/responses/search/search-response'; -import { MetadataResponse } from '../src/responses/metadata/metadata-response'; import { Result } from '@internetarchive/result-type'; import { SearchServiceError, SearchServiceErrorType, } from '../src/search-service-error'; import { SearchBackendInterface } from '../src/search-backend/search-backend-interface'; +import { SearchType } from '../src/search-type'; +import { MetadataSearchBackend } from '../src/search-backend/metadata-search-backend'; +import { FulltextSearchBackend } from '../src/search-backend/fulltext-search-backend'; describe('SearchService', () => { it('can search when requested', async () => { class MockSearchBackend implements SearchBackendInterface { - async fetchMetadata( - identifier: string - ): Promise> { - throw new Error('Method not implemented.'); - } - async performSearch( params: SearchParams - ): Promise> { + ): Promise> { const responseGenerator = new MockResponseGenerator(); - const mockResponse = responseGenerator.generateMockSearchResponse( + const mockResponse = responseGenerator.generateMockMetadataSearchResponse( params ); return { success: mockResponse }; } } - const query = 'title:foo AND collection:bar'; - const backend = new MockSearchBackend(); - const service = new SearchService(backend); - const result = await service.search({ query }); - expect(result.success?.responseHeader.params.query).to.equal(query); - }); - - it('can request metadata when requested', async () => { - class MockSearchBackend implements SearchBackendInterface { - performSearch( - params: SearchParams - ): Promise> { - throw new Error('Method not implemented.'); - } - async fetchMetadata( - identifier: string - ): Promise> { - const responseGenerator = new MockResponseGenerator(); - const mockResponse = responseGenerator.generateMockMetadataResponse( - identifier - ); - return { success: mockResponse }; - } - } - const backend = new MockSearchBackend(); - const service = new SearchService(backend); - const result = await service.fetchMetadata('foo'); - expect(result.success?.metadata.identifier).to.equal('foo'); - }); + const realFactoryMethod = SearchService.getBackendForSearchType; + SearchService.getBackendForSearchType = () => backend; - describe('requestMetadataValue', async () => { - class MockSearchBackend implements SearchBackendInterface { - response: any; - performSearch( - params: SearchParams - ): Promise> { - throw new Error('Method not implemented.'); - } - async fetchMetadata( - identifier: string, - keypath?: string - ): Promise> { - return { - success: { - result: this.response, - }, - }; - } - } - - it('can request a metadata value', async () => { - const backend = new MockSearchBackend(); - const service = new SearchService(backend); - - let expectedResult: any = 'foo'; - backend.response = expectedResult; - - let result = await service.fetchMetadataValue( - 'foo', - 'metadata' - ); - expect(result.success).to.equal(expectedResult); - - expectedResult = { foo: 'bar' }; - backend.response = expectedResult; - - result = await service.fetchMetadataValue( - 'foo', - 'metadata' - ); - expect(result.success).to.equal(expectedResult); - expect(result.success.foo).to.equal('bar'); - }); - }); - - it('returns an error result if the item is not found', async () => { - class MockSearchBackend implements SearchBackendInterface { - performSearch( - params: SearchParams - ): Promise> { - throw new Error('Method not implemented.'); - } - async fetchMetadata( - identifier: string - ): Promise> { - // this is unfortunate.. instead of getting an http 404 error, - // we get an empty JSON object when an item is not found - return { success: {} as any }; - } - } - - const backend = new MockSearchBackend(); - const service = new SearchService(backend); - const result = await service.fetchMetadata('foo'); - expect(result.error).to.not.equal(undefined); - expect(result.error?.type).to.equal(SearchServiceErrorType.itemNotFound); - - const valueResult = await service.fetchMetadataValue('foo', 'metadata'); - expect(valueResult.error).to.not.equal(undefined); - expect(valueResult.error?.type).to.equal( - SearchServiceErrorType.itemNotFound + const query = 'title:foo AND collection:bar'; + const service = new SearchService(); + const result = await service.search({ query }); + expect(result.success?.request.finalizedParameters.user_query).to.equal( + query ); + + SearchService.getBackendForSearchType = realFactoryMethod; }); it('returns the search backend network error if one occurs', async () => { class MockSearchBackend implements SearchBackendInterface { async performSearch( params: SearchParams - ): Promise> { - const error = new SearchServiceError( - SearchServiceErrorType.networkError, - 'network error' - ); - return { error }; - } - async fetchMetadata( - identifier: string - ): Promise> { + ): Promise> { const error = new SearchServiceError( SearchServiceErrorType.networkError, 'network error' @@ -163,20 +58,10 @@ describe('SearchService', () => { } const backend = new MockSearchBackend(); - const service = new SearchService(backend); - const metadataResult = await service.fetchMetadata('foo'); - expect(metadataResult.error).to.not.equal(undefined); - expect(metadataResult.error?.type).to.equal( - SearchServiceErrorType.networkError - ); - expect(metadataResult.error?.message).to.equal('network error'); + const realFactoryMethod = SearchService.getBackendForSearchType; + SearchService.getBackendForSearchType = () => backend; - const metadataValueResult = await service.fetchMetadataValue('foo', 'bar'); - expect(metadataValueResult.error).to.not.equal(undefined); - expect(metadataValueResult.error?.type).to.equal( - SearchServiceErrorType.networkError - ); - expect(metadataValueResult.error?.message).to.equal('network error'); + const service = new SearchService(); const searchResult = await service.search({ query: 'boop' }); expect(searchResult.error).to.not.equal(undefined); @@ -184,22 +69,15 @@ describe('SearchService', () => { SearchServiceErrorType.networkError ); expect(searchResult.error?.message).to.equal('network error'); + + SearchService.getBackendForSearchType = realFactoryMethod; }); it('returns the search backend decoding error if one occurs', async () => { class MockSearchBackend implements SearchBackendInterface { async performSearch( params: SearchParams - ): Promise> { - const error = new SearchServiceError( - SearchServiceErrorType.decodingError, - 'decoding error' - ); - return { error }; - } - async fetchMetadata( - identifier: string - ): Promise> { + ): Promise> { const error = new SearchServiceError( SearchServiceErrorType.decodingError, 'decoding error' @@ -209,13 +87,10 @@ describe('SearchService', () => { } const backend = new MockSearchBackend(); - const service = new SearchService(backend); - const metadataResult = await service.fetchMetadata('foo'); - expect(metadataResult.error).to.not.equal(undefined); - expect(metadataResult.error?.type).to.equal( - SearchServiceErrorType.decodingError - ); - expect(metadataResult.error?.message).to.equal('decoding error'); + const realFactoryMethod = SearchService.getBackendForSearchType; + SearchService.getBackendForSearchType = () => backend; + + const service = new SearchService(); const searchResult = await service.search({ query: 'boop' }); expect(searchResult.error).to.not.equal(undefined); @@ -223,5 +98,58 @@ describe('SearchService', () => { SearchServiceErrorType.decodingError ); expect(searchResult.error?.message).to.equal('decoding error'); + + SearchService.getBackendForSearchType = realFactoryMethod; + }); + + it('passes backend options to backend', async () => { + class MockSearchBackend implements SearchBackendInterface { + async performSearch( + params: SearchParams + ): Promise> { + const responseGenerator = new MockResponseGenerator(); + const mockResponse = responseGenerator.generateMockMetadataSearchResponse( + params + ); + return { success: mockResponse }; + } + } + + const backend = new MockSearchBackend(); + const spy = sinon.spy(); + + const realFactoryMethod = SearchService.getBackendForSearchType; + SearchService.getBackendForSearchType = (...args) => { + spy(...args); + return backend; + }; + + const backendOptions = { + baseUrl: 'foo.bar', + includeCredentials: true, + scope: 'baz', + }; + + const service = new SearchService(backendOptions); + + const params = { query: 'boop' }; + await service.search(params); + + expect(spy.callCount).to.equal(1); + expect(spy.calledWithExactly(params, backendOptions)); + + SearchService.getBackendForSearchType = realFactoryMethod; + }); + + it('factory method gets metadata backend', async () => { + expect( + SearchService.getBackendForSearchType(SearchType.METADATA) + ).to.be.instanceOf(MetadataSearchBackend); + }); + + it('factory method gets fulltext backend', async () => { + expect( + SearchService.getBackendForSearchType(SearchType.FULLTEXT) + ).to.be.instanceOf(FulltextSearchBackend); }); }); diff --git a/tsconfig.json b/tsconfig.json index 5736b1d5..70cf89bf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "module": "esnext", "moduleResolution": "node", "noEmitOnError": true, - "lib": ["es2017", "dom"], + "lib": ["es2017", "dom", "dom.iterable"], "strict": true, "esModuleInterop": false, "allowSyntheticDefaultImports": true,