forked from mhujer/ankiai
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Change from batch to concurrent batch processing
- Loading branch information
Showing
10 changed files
with
166 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
import { NoteForProcessing } from "../anki"; | ||
import { VocabularyExamples } from "../openai/typechat-response-schema"; | ||
import { removeSentencesWithoutCharacter } from "../utils/remove-sentences-without-character"; | ||
|
||
const capitaliseFirstLetter = (str: string): string => { | ||
return str.charAt(0).toUpperCase() + str.slice(1); | ||
} | ||
|
||
export const formatExamplesForAnki = (examples: VocabularyExamples[], note: NoteForProcessing): string[] => { | ||
const ankiExamples: string[] = []; | ||
for (const example of examples) { | ||
// Remove sentences that do not contain the target word | ||
const filteredExampleSentences = removeSentencesWithoutCharacter(example.exampleSentences, note.text); | ||
if (example.exampleSentences.length !== filteredExampleSentences.length) { | ||
for (const exampleSentence of example.exampleSentences) { | ||
if (!example.exampleSentences.includes(note.text)) { | ||
console.error(`${note.text} is not present in ${exampleSentence}.`); | ||
console.error( | ||
`Adding ${filteredExampleSentences.length} examples to card, filtered from ${example.exampleSentences.length}.`, | ||
); | ||
} | ||
} | ||
} | ||
|
||
// Only add the part of speech section if there are examples | ||
if (filteredExampleSentences.length > 0) { | ||
ankiExamples.push( | ||
`<div class="char_examples-parts-of-speech">${capitaliseFirstLetter(example.partsOfSpeech)}</div>`, | ||
); | ||
for (const exampleSentence of filteredExampleSentences) { | ||
ankiExamples.push(`<div class="char_example">${exampleSentence}</div>`); | ||
} | ||
} | ||
} | ||
|
||
return ankiExamples; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
import { describe, expect, jest, afterEach, it } from '@jest/globals'; | ||
import { logger } from './logger'; | ||
import { concurrentProcessor } from './concurrent-processor'; | ||
|
||
// Mock logger | ||
jest.mock('./logger', () => ({ | ||
logger: { | ||
info: jest.fn(), | ||
debug: jest.fn() | ||
} | ||
})); | ||
|
||
const mockProcessItem = jest.fn().mockImplementation((item: any) => { | ||
return Promise.resolve(); | ||
}); | ||
|
||
describe('concurrentProcessor function', () => { | ||
afterEach(() => { | ||
jest.clearAllMocks(); | ||
}); | ||
|
||
it('should process all items without processing the same item twice', async () => { | ||
const items = [1, 2, 3, 4, 5]; | ||
|
||
await concurrentProcessor(items, 2, mockProcessItem as any); | ||
|
||
expect(logger.info).toHaveBeenCalledWith('Starting to process 5 items'); | ||
expect(logger.debug).toHaveBeenCalledTimes(5 * 2); // 5 items, each processed twice | ||
expect(mockProcessItem).toHaveBeenCalledTimes(5); // 5 items processed | ||
}); | ||
|
||
it('should handle concurrency limit properly', async () => { | ||
const items = [1, 2, 3, 4, 5]; | ||
|
||
await concurrentProcessor(items, 2, mockProcessItem as any); | ||
|
||
expect(logger.info).toHaveBeenCalledWith('Starting to process 5 items'); | ||
expect(logger.debug).toHaveBeenCalledTimes(5 * 2); // 5 items, each processed twice | ||
expect(mockProcessItem).toHaveBeenCalledTimes(5); // 5 items processed | ||
|
||
// Ensure that only 2 items were processed concurrently | ||
expect(mockProcessItem.mock.calls.slice(0, 2)).toEqual([[1], [2]]); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
import { logger } from "./logger"; | ||
import { ProcessNote } from "./process-note"; | ||
|
||
// Define a generic function to process items with a concurrency limit | ||
export async function concurrentProcessor<T>( | ||
items: T[], | ||
concurrencyLimit: number, | ||
processItem: (item: T) => Promise<void> | ||
): Promise<void> { | ||
logger.info(`Starting to process ${items.length} items`); | ||
const processingQueue: Promise<void>[] = []; | ||
|
||
async function processNextItem(): Promise<void> { | ||
const item = items.shift(); | ||
if (item) { | ||
logger.debug(`Starting to process item: ${JSON.stringify(item)}`); | ||
|
||
const processingPromise = processItem(item); | ||
processingQueue.push(processingPromise); | ||
await processingPromise; | ||
const index = processingQueue.indexOf(processingPromise); | ||
if (index !== -1) { | ||
processingQueue.splice(index, 1); | ||
} | ||
console.log(`${items.length} items remaining, completed processing item: ${JSON.stringify(item)}`); | ||
logger.debug(`${items.length} items remaining, completed processing item: ${JSON.stringify(item)}`); | ||
await processNextItem(); | ||
} | ||
} | ||
|
||
// Start initial tasks up to the concurrency limit | ||
for (let i = 0; i < concurrencyLimit && items.length > 0; i++) { | ||
processingQueue.push(processNextItem()); | ||
} | ||
|
||
// Wait for all tasks to complete | ||
await Promise.all(processingQueue); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import { NoteForProcessing, updateNote } from "../anki"; | ||
import { formatExamplesForAnki } from "../anki/format-items-for-anki"; | ||
import { fetchExample } from "../openai/get-sentences-from-chatgpt"; | ||
import { logger } from "./logger"; | ||
import sleep from "./sleep"; | ||
|
||
export interface ProcessNote { | ||
(noteForProcessing: NoteForProcessing): Promise<void>; | ||
} | ||
|
||
export const primeProcessNote = (ankiLanguage: string): ProcessNote => { | ||
return async (noteForProcessing: NoteForProcessing): Promise<void> => { | ||
const examplesFromChatGPT = await fetchExample(ankiLanguage, noteForProcessing); | ||
|
||
const fieldText = formatExamplesForAnki(examplesFromChatGPT, noteForProcessing).join(''); | ||
|
||
const noteId = noteForProcessing.noteId; | ||
const noteForAnki = { | ||
id: +noteId, | ||
fields: { | ||
Examples: fieldText, | ||
}, | ||
}; | ||
|
||
await updateNote(noteForAnki); | ||
logger.debug(`Updated Anki with ${noteForProcessing.text}`); | ||
return await sleep(250); | ||
}; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters