Skip to content

Commit

Permalink
Merge ebf537e into b659edd
Browse files Browse the repository at this point in the history
  • Loading branch information
lucaong committed Nov 27, 2022
2 parents b659edd + ebf537e commit 4aa1428
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 8 deletions.
26 changes: 26 additions & 0 deletions src/MiniSearch.test.js
Expand Up @@ -759,6 +759,32 @@ describe('MiniSearch', () => {
})
})

describe('addFields', () => {
it('add fields to an existing document', () => {
const options = { fields: ['text', 'author'], storeFields: ['text', 'author', 'n'] }
const ms = new MiniSearch(options)
const other = new MiniSearch(options)

ms.add({ id: 1, text: 'Some quite interesting stuff' })
ms.addFields(1, { author: 'Al et. al.', n: 5 })

other.add({ id: 1, text: 'Some quite interesting stuff', author: 'Al et. al.', n: 5 })

expect(ms).toEqual(other)
})

it('throws an error if the document did not exist', () => {
const ms = new MiniSearch({ fields: ['text'] })
expect(() => { ms.addFields(1, { text: 'hello' }) }).toThrow('MiniSearch: no document with ID 1')
})

it('throws an error if adding a field that already exists', () => {
const ms = new MiniSearch({ fields: ['text'] })
ms.add({ id: 1, text: 'Some interesting stuff' })
expect(() => { ms.addFields(1, { text: 'hello' }) }).toThrow('MiniSearch: field text already exists on document with ID 1')
})
})

describe('vacuum', () => {
it('cleans up discarded documents from the index', async () => {
const ms = new MiniSearch({ fields: ['text'], storeFields: ['text'] })
Expand Down
70 changes: 62 additions & 8 deletions src/MiniSearch.ts
Expand Up @@ -588,7 +588,7 @@ export default class MiniSearch<T = any> {
this._enqueuedVacuum = null
this._enqueuedVacuumConditions = defaultVacuumConditions

this.addFields(this._options.fields)
this.addFieldIds(this._options.fields)
}

/**
Expand All @@ -597,8 +597,9 @@ export default class MiniSearch<T = any> {
* @param document The document to be indexed
*/
add (document: T): void {
const { extractField, tokenize, processTerm, fields, idField } = this._options
const { extractField, idField } = this._options
const id = extractField(document, idField)

if (id == null) {
throw new Error(`MiniSearch: document does not have ID field "${idField}"`)
}
Expand All @@ -610,15 +611,66 @@ export default class MiniSearch<T = any> {
const shortDocumentId = this.addDocumentId(id)
this.saveStoredFields(shortDocumentId, document)

this.addToIndex(shortDocumentId, document, true)
}

/**
* Adds some fields to an existing documeny
*
* The added fields should not be already present on the document, or an error
* will be thrown.
*
* ## Example:
*
* const miniSearch = new MiniSearch({ fields: ['title', 'text', 'author'] })
*
* miniSearch.add({ id: 1, title: 'Neuromancer' })
*
* miniSearch.addFields(1, {
* text: 'The sky above the port was the color of television, tuned to a dead channel.',
* author: 'William Gibson'
* })
*
* // The above is equivalent to:
* miniSearch.add({
* id: 1,
* title: 'Neuromancer',
* text: 'The sky above the port was the color of television, tuned to a dead channel.',
* author: 'William Gibson'
* })
*
* @param id The document ID
* @param toAdd The fields to add
*/
addFields (id: any, toAdd: T): void {
const shortDocumentId = this._idToShortId.get(id)

if (shortDocumentId == null) {
throw new Error(`MiniSearch: no document with ID ${id}`)
}

this.saveStoredFields(shortDocumentId, toAdd)

this.addToIndex(shortDocumentId, toAdd, false)
}

private addToIndex (shortDocumentId: number, document: T, added: boolean) {
const { extractField, tokenize, processTerm, fields } = this._options

for (const field of fields) {
const fieldValue = extractField(document, field)
if (fieldValue == null) continue

const tokens = tokenize(fieldValue.toString(), field)
const fieldId = this._fieldIds[field]

const uniqueTerms = new Set(tokens).size
this.addFieldLength(shortDocumentId, fieldId, this._documentCount - 1, uniqueTerms)
const uniqueTerms = new Set(tokens)
uniqueTerms.delete('')

if (this._fieldLength.get(shortDocumentId)?.[fieldId] != null) {
throw new Error(`MiniSearch: field ${field} already exists on document with ID ${this._documentIds.get(shortDocumentId)}`)
}
this.addFieldLength(shortDocumentId, fieldId, this._documentCount, uniqueTerms.size, added)

for (const term of tokens) {
const processedTerm = processTerm(term, field)
Expand Down Expand Up @@ -1706,7 +1758,7 @@ export default class MiniSearch<T = any> {
/**
* @ignore
*/
private addFields (fields: string[]): void {
private addFieldIds (fields: string[]): void {
for (let i = 0; i < fields.length; i++) {
this._fieldIds[fields[i]] = i
}
Expand All @@ -1715,14 +1767,16 @@ export default class MiniSearch<T = any> {
/**
* @ignore
*/
private addFieldLength (documentId: number, fieldId: number, count: number, length: number): void {
private addFieldLength (documentId: number, fieldId: number, count: number, length: number, added: boolean): void {
let fieldLengths = this._fieldLength.get(documentId)
if (fieldLengths == null) this._fieldLength.set(documentId, fieldLengths = [])
const n = added ? 1 : 0

fieldLengths[fieldId] = length

const averageFieldLength = this._avgFieldLength[fieldId] || 0
const totalFieldLength = (averageFieldLength * count) + length
this._avgFieldLength[fieldId] = totalFieldLength / (count + 1)
const totalFieldLength = (averageFieldLength * (count - n)) + length
this._avgFieldLength[fieldId] = totalFieldLength / count
}

/**
Expand Down

0 comments on commit 4aa1428

Please sign in to comment.