Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 54 additions & 2 deletions desktop/src/lib/importSummary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,19 +147,71 @@ export function summarizeImportFiles(
};
}

/** Run async work on `items` with at most `concurrency` in flight (order preserved). */
export async function mapPool<T, R>(
items: readonly T[],
concurrency: number,
worker: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const n = items.length;
if (n === 0) {
return [];
}
const results: R[] = new Array(n);
let next = 0;
const limit = Math.max(1, Math.min(concurrency, n));

async function slot(): Promise<void> {
while (true) {
const i = next++;
if (i >= n) {
return;
}
results[i] = await worker(items[i], i);
}
}

await Promise.all(Array.from({ length: limit }, () => slot()));
return results;
}

const DEFAULT_JSON_READ_CONCURRENCY = 12;

export async function parseObservationJsonFiles(
files: readonly File[],
concurrency = DEFAULT_JSON_READ_CONCURRENCY,
): Promise<ParsedObservationFile[]> {
return mapPool(files, concurrency, (file, _i) =>
parseObservationJsonFile(file),
);
}

export async function parseObservationJsonFile(
file: File,
): Promise<ParsedObservationFile> {
const fileName = file.name;
let text: string;
try {
text = await file.text();
} catch (e) {
const hint =
e instanceof Error && e.name === 'NotReadableError'
? ' (try validating again; parallel read limit)'
: '';
return {
fileName,
observations: [],
error: `Could not read file${hint}`,
};
}
try {
const text = await file.text();
const root = JSON.parse(text) as unknown;
const { observations, error } = extractObservationsFromJson(root, fileName);
if (error) {
return { fileName, observations: [], error };
}
return { fileName, observations };
} catch {
} catch (_e) {
return { fileName, observations: [], error: 'Invalid JSON' };
}
}
Expand Down
6 changes: 3 additions & 3 deletions desktop/src/pages/ImportPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
import type { BundleFormSpec } from '../types/domain';
import {
flattenObservations,
parseObservationJsonFile,
parseObservationJsonFiles,
summarizeImportFiles,
} from '../lib/importSummary';
import {
Expand Down Expand Up @@ -117,8 +117,8 @@ export function ImportPage() {
setMessage(null);
setError(null);
try {
const parsed = await Promise.all(
stagedJson.map(s => parseObservationJsonFile(s.file)),
const parsed = await parseObservationJsonFiles(
stagedJson.map(s => s.file),
);
const formTypes = new Set<string>();
for (const p of parsed) {
Expand Down
Loading