From 83a92036b3cdb65f785695bb6ae0fbd9dab3367e Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 16:08:06 +0100 Subject: [PATCH 01/14] feat: backfill cv to profile --- bin/importProfileFromJSON.ts | 86 ++++++++++++++++--- src/common/profile/import.ts | 31 +++++-- src/common/schema/profile.ts | 68 ++++++++++----- src/entity/user/experiences/UserExperience.ts | 7 ++ ...1763996658211-UserExperienceFlagsImport.ts | 19 ++++ 5 files changed, 172 insertions(+), 39 deletions(-) create mode 100644 src/migration/1763996658211-UserExperienceFlagsImport.ts diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index c792780f09..8ec7291ee4 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -3,9 +3,10 @@ import '../src/config'; import { parseArgs } from 'node:util'; import { z } from 'zod'; import createOrGetConnection from '../src/db'; -import { type DataSource } from 'typeorm'; -import { readFile } from 'node:fs/promises'; +import { QueryFailedError, type DataSource } from 'typeorm'; +import { readFile, stat, readdir } from 'node:fs/promises'; import { importUserExperienceFromJSON } from '../src/common/profile/import'; +import path from 'node:path'; /** * Import profile from JSON to user by id @@ -14,6 +15,7 @@ import { importUserExperienceFromJSON } from '../src/common/profile/import'; */ const main = async () => { let con: DataSource | null = null; + let failedImports = 0; try { const { values } = parseArgs({ @@ -22,29 +24,87 @@ const main = async () => { type: 'string', short: 'p', }, - userId: { + limit: { type: 'string', - short: 'u', + short: 'l', }, }, }); const paramsSchema = z.object({ path: z.string().nonempty(), - userId: z.string().nonempty(), + limit: z.coerce.number().int().positive().default(10), }); const params = paramsSchema.parse(values); con = await createOrGetConnection(); - const dataJSON = JSON.parse(await readFile(params.path, 'utf-8')); + const pathStat = await stat(params.path); - await importUserExperienceFromJSON({ - con: con.manager, - dataJson: dataJSON, - userId: params.userId, - }); + let filePaths = [params.path]; + + if (pathStat.isDirectory()) { + filePaths = await readdir(params.path); + } + + console.log('Found files:', filePaths.length); + + console.log( + `Importing:`, + Math.min(params.limit, filePaths.length), + `(limit ${params.limit})`, + ); + + for (const [index, fileName] of filePaths + .slice(0, params.limit) + .entries()) { + const filePath = + params.path === fileName ? fileName : path.join(params.path, fileName); + + try { + if (!filePath.endsWith('.json')) { + throw { type: 'not_json_ext', filePath }; + } + + const userId = filePath.split('/').pop()?.split('.json')[0]; + + if (!userId) { + throw { type: 'no_user_id', filePath }; + } + + const dataJSON = JSON.parse(await readFile(filePath, 'utf-8')); + + await importUserExperienceFromJSON({ + con: con.manager, + dataJson: dataJSON, + userId: 'testuser', + }); + } catch (error) { + failedImports += 1; + + if (error instanceof QueryFailedError) { + console.error({ + type: 'db_query_failed', + message: error.message, + filePath, + }); + } else if (error instanceof z.ZodError) { + console.error({ + type: 'zod_error', + message: error.issues[0].message, + path: error.issues[0].path, + filePath, + }); + } else { + console.error(error); + } + } + + if (index && index % 100 === 0) { + console.log('Done so far:', index, ', failed:', failedImports); + } + } } catch (error) { console.error(error instanceof z.ZodError ? z.prettifyError(error) : error); } finally { @@ -52,6 +112,10 @@ const main = async () => { con.destroy(); } + if (failedImports > 0) { + console.log(`Failed imports: ${failedImports}`); + } + process.exit(0); } }; diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index 8c3089395f..f578cfb4f6 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -58,7 +58,11 @@ const resolveUserLocationPart = async ({ con, threshold = 0.5, }: { - location?: Partial>; + location?: { + city?: string | null; + subdivision?: string | null; + country?: string | null; + } | null; con: EntityManager; threshold?: number; }): Promise>> => { @@ -324,12 +328,25 @@ export const importUserExperienceFromJSON = async ({ } const data = z - .array( - userExperienceInputBaseSchema - .pick({ - type: true, - }) - .loose(), + .preprocess( + (item) => { + if (item === null) { + return []; + } + + if (typeof item === 'object' && !Array.isArray(item)) { + return []; + } + + return item; + }, + z.array( + userExperienceInputBaseSchema + .pick({ + type: true, + }) + .loose(), + ), ) .parse(dataJson); diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index 09e26ef455..d0cdffa9b8 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -100,51 +100,77 @@ export const getExperienceSchema = (type: UserExperienceType) => { export const userExperienceWorkImportSchema = z.object({ type: z.string(), - company: z.string(), - title: z.string(), - description: z.string().optional(), + company: z.string().nullish(), + title: z + .string() + .nullish() + .transform((n) => (n === null ? undefined : n)) + .default('Work experience'), + description: z.string().nullish(), started_at: z.coerce.date().default(() => new Date()), - location_type: z.string().optional(), - skills: z.array(z.string()).optional(), + location_type: z.string().nullish(), + skills: z + .array(z.string()) + .nullish() + .transform((n) => (n === null ? undefined : n)) + .default([]), ended_at: z.coerce.date().nullish().default(null), location: z .object({ - city: z.string().optional(), - country: z.string(), + city: z.string().nullish(), + country: z.string().nullish(), }) - .optional(), + .nullish(), }); export const userExperienceEducationImportSchema = z.object({ type: z.string(), - company: z.string().optional(), - title: z.string(), - description: z.string().optional(), + company: z.string().nullish(), + title: z + .string() + .nullish() + .transform((n) => (n === null ? undefined : n)) + .default('Education'), + description: z.string().nullish(), started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), location: z .object({ - city: z.string().optional(), - country: z.string(), + city: z.string().nullish(), + country: z.string().nullish(), }) - .optional(), - skills: z.array(z.string()).optional(), - subtitle: z.string().optional(), + .nullish(), + skills: z + .array(z.string()) + .nullish() + .transform((n) => (n === null ? undefined : n)), + subtitle: z.string().nullish(), }); export const userExperienceCertificationImportSchema = z.object({ type: z.string(), - company: z.string().optional(), - title: z.string(), + company: z.string().nullish(), + title: z + .string() + .nullish() + .transform((n) => (n === null ? undefined : n)) + .default('Certification'), started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), }); export const userExperienceProjectImportSchema = z.object({ type: z.string(), - title: z.string(), - description: z.string(), + title: z + .string() + .nullish() + .transform((n) => (n === null ? undefined : n)) + .default('Project'), + description: z.string().nullish(), started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), - skills: z.array(z.string()), + skills: z + .array(z.string()) + .nullish() + .transform((n) => (n === null ? undefined : n)), }); diff --git a/src/entity/user/experiences/UserExperience.ts b/src/entity/user/experiences/UserExperience.ts index 797f59f3e4..8df32d5d61 100644 --- a/src/entity/user/experiences/UserExperience.ts +++ b/src/entity/user/experiences/UserExperience.ts @@ -15,6 +15,10 @@ import type { Company } from '../../Company'; import { LocationType } from '@dailydotdev/schema'; import type { DatasetLocation } from '../../dataset/DatasetLocation'; +export type UserExperienceFlags = Partial<{ + import: string; +}>; + @Entity() @TableInheritance({ column: { type: 'text', name: 'type' } }) export class UserExperience { @@ -84,4 +88,7 @@ export class UserExperience { @UpdateDateColumn({ type: 'timestamp' }) updatedAt: Date; + + @Column({ type: 'jsonb', default: {} }) + flags: UserExperienceFlags; } diff --git a/src/migration/1763996658211-UserExperienceFlagsImport.ts b/src/migration/1763996658211-UserExperienceFlagsImport.ts new file mode 100644 index 0000000000..04c52ffd9d --- /dev/null +++ b/src/migration/1763996658211-UserExperienceFlagsImport.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UserExperienceFlagsImport1763996658211 + implements MigrationInterface +{ + name = 'UserExperienceFlagsImport1763996658211'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "user_experience" ADD "flags" jsonb NOT NULL DEFAULT '{}'`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "user_experience" DROP COLUMN "flags"`, + ); + } +} From 7679f0b13f0f69b258a69884d300f7f4e359d72e Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 16:28:39 +0100 Subject: [PATCH 02/14] feat: add support for import id --- bin/importProfileFromJSON.ts | 8 ++++++++ src/common/profile/import.ts | 27 +++++++++++++++++++++------ src/common/schema/profile.ts | 4 ++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index 8ec7291ee4..aa006a7322 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -7,6 +7,7 @@ import { QueryFailedError, type DataSource } from 'typeorm'; import { readFile, stat, readdir } from 'node:fs/promises'; import { importUserExperienceFromJSON } from '../src/common/profile/import'; import path from 'node:path'; +import { randomUUID } from 'node:crypto'; /** * Import profile from JSON to user by id @@ -28,16 +29,22 @@ const main = async () => { type: 'string', short: 'l', }, + uid: { + type: 'string', + }, }, }); const paramsSchema = z.object({ path: z.string().nonempty(), limit: z.coerce.number().int().positive().default(10), + uid: z.string().nonempty().default(randomUUID()), }); const params = paramsSchema.parse(values); + console.log('Starting import with ID:', params.uid); + con = await createOrGetConnection(); const pathStat = await stat(params.path); @@ -79,6 +86,7 @@ const main = async () => { con: con.manager, dataJson: dataJSON, userId: 'testuser', + importId: params.uid, }); } catch (error) { failedImports += 1; diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index f578cfb4f6..d7d715a0f5 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -141,10 +141,12 @@ export const importUserExperienceWork = async ({ skills, ended_at: endedAt, location, + flags, } = userExperience; const insertResult = await con.getRepository(UserExperienceWork).insert( con.getRepository(UserExperienceWork).create({ + flags, userId: userId, ...(await resolveUserCompanyPart({ name: company, @@ -202,10 +204,12 @@ export const importUserExperienceEducation = async ({ ended_at: endedAt, location, subtitle, + flags, } = userExperience; const insertResult = await con.getRepository(UserExperienceEducation).insert( con.getRepository(UserExperienceEducation).create({ + flags, userId: userId, ...(await resolveUserCompanyPart({ name: company, @@ -250,12 +254,14 @@ export const importUserExperienceCertification = async ({ title, started_at: startedAt, ended_at: endedAt, + flags, } = userExperience; const insertResult = await con .getRepository(UserExperienceCertification) .insert( con.getRepository(UserExperienceCertification).create({ + flags, userId: userId, ...(await resolveUserCompanyPart({ name: company, @@ -291,10 +297,12 @@ export const importUserExperienceProject = async ({ started_at: startedAt, ended_at: endedAt, skills, + flags, } = userExperience; const insertResult = await con.getRepository(UserExperienceProject).insert( con.getRepository(UserExperienceProject).create({ + flags, userId: userId, title, description, @@ -318,10 +326,12 @@ export const importUserExperienceFromJSON = async ({ con, dataJson, userId, + importId, }: { con: EntityManager; dataJson: unknown; userId: string; + importId?: string; }) => { if (!userId) { throw new Error('userId is required'); @@ -352,10 +362,15 @@ export const importUserExperienceFromJSON = async ({ await con.transaction(async (entityManager) => { for (const item of data) { - switch (item.type) { + const importData = { + ...item, + flags: importId ? { import: importId } : undefined, + }; + + switch (importData.type) { case UserExperienceType.Work: await importUserExperienceWork({ - data: item, + data: importData, con: entityManager, userId, }); @@ -363,7 +378,7 @@ export const importUserExperienceFromJSON = async ({ break; case UserExperienceType.Education: await importUserExperienceEducation({ - data: item, + data: importData, con: entityManager, userId, }); @@ -371,7 +386,7 @@ export const importUserExperienceFromJSON = async ({ break; case UserExperienceType.Certification: await importUserExperienceCertification({ - data: item, + data: importData, con: entityManager, userId, }); @@ -381,14 +396,14 @@ export const importUserExperienceFromJSON = async ({ case UserExperienceType.OpenSource: case UserExperienceType.Volunteering: await importUserExperienceProject({ - data: item, + data: importData, con: entityManager, userId, }); break; default: - throw new Error(`Unsupported experience type: ${item.type}`); + throw new Error(`Unsupported experience type: ${importData.type}`); } } }); diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index d0cdffa9b8..04dafc67f7 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -121,6 +121,7 @@ export const userExperienceWorkImportSchema = z.object({ country: z.string().nullish(), }) .nullish(), + flags: z.object({ import: z.string() }).partial().optional(), }); export const userExperienceEducationImportSchema = z.object({ @@ -145,6 +146,7 @@ export const userExperienceEducationImportSchema = z.object({ .nullish() .transform((n) => (n === null ? undefined : n)), subtitle: z.string().nullish(), + flags: z.object({ import: z.string() }).partial().optional(), }); export const userExperienceCertificationImportSchema = z.object({ @@ -157,6 +159,7 @@ export const userExperienceCertificationImportSchema = z.object({ .default('Certification'), started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), + flags: z.object({ import: z.string() }).partial().optional(), }); export const userExperienceProjectImportSchema = z.object({ @@ -173,4 +176,5 @@ export const userExperienceProjectImportSchema = z.object({ .array(z.string()) .nullish() .transform((n) => (n === null ? undefined : n)), + flags: z.object({ import: z.string() }).partial().optional(), }); From 57f8523273e1a157386ac7fa1d15bdceeae70ab3 Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 16:31:08 +0100 Subject: [PATCH 03/14] fix: tests --- __tests__/common/profile/import.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/__tests__/common/profile/import.ts b/__tests__/common/profile/import.ts index 56c7348ea6..82e2eb64ce 100644 --- a/__tests__/common/profile/import.ts +++ b/__tests__/common/profile/import.ts @@ -59,6 +59,7 @@ describe('UserExperienceType work import', () => { verified: false, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); const skills = await con .getRepository(UserExperienceSkill) @@ -96,6 +97,7 @@ describe('UserExperienceType work import', () => { updatedAt: expect.any(Date), userId: 'user-work-2', verified: false, + flags: {}, }); }); }); @@ -131,6 +133,7 @@ describe('UserExperienceType education import', () => { grade: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); }); @@ -163,6 +166,7 @@ describe('UserExperienceType education import', () => { grade: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); }); }); @@ -200,6 +204,7 @@ describe('UserExperienceType certification import', () => { url: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); }); @@ -235,6 +240,7 @@ describe('UserExperienceType certification import', () => { url: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); }); }); @@ -273,6 +279,7 @@ describe('UserExperienceType project import', () => { url: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); expect(skills.map((s) => s.value).sort()).toEqual( ['GraphQL', 'Node.js'].sort(), @@ -308,6 +315,7 @@ describe('UserExperienceType project import', () => { url: null, createdAt: expect.any(Date), updatedAt: expect.any(Date), + flags: {}, }); }); }); From c0dd063b8b8ecaabd9527b5f093efbf8ed55c999 Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 16:38:06 +0100 Subject: [PATCH 04/14] feat: add offset support --- bin/importProfileFromJSON.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index aa006a7322..d16ba38df0 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -29,6 +29,10 @@ const main = async () => { type: 'string', short: 'l', }, + offset: { + type: 'string', + short: 'o', + }, uid: { type: 'string', }, @@ -38,6 +42,7 @@ const main = async () => { const paramsSchema = z.object({ path: z.string().nonempty(), limit: z.coerce.number().int().positive().default(10), + offset: z.coerce.number().int().positive().default(0), uid: z.string().nonempty().default(randomUUID()), }); @@ -52,9 +57,11 @@ const main = async () => { let filePaths = [params.path]; if (pathStat.isDirectory()) { - filePaths = await readdir(params.path); + filePaths = await readdir(params.path, 'utf-8'); } + filePaths.sort(); // ensure consistent order for offset/limit + console.log('Found files:', filePaths.length); console.log( @@ -64,7 +71,7 @@ const main = async () => { ); for (const [index, fileName] of filePaths - .slice(0, params.limit) + .slice(params.offset, params.offset + params.limit) .entries()) { const filePath = params.path === fileName ? fileName : path.join(params.path, fileName); From aac3277a03914544329094743bb39854ba5d6fe1 Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 16:57:35 +0100 Subject: [PATCH 05/14] fix: similarity quote escape --- src/common/profile/import.ts | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index d7d715a0f5..e417fdfe87 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -37,8 +37,9 @@ const resolveUserCompanyPart = async ({ const company = await con .getRepository(Company) .createQueryBuilder() + .setParameter('companyName', name) .addSelect('id') - .addSelect(`similarity(name, '${name}')`, 'similarity') + .addSelect(`similarity(name, :companyName)`, 'similarity') .orderBy('similarity', 'DESC') .getRawOne & { similarity: number }>(); @@ -76,26 +77,32 @@ const resolveUserLocationPart = async ({ .addSelect('id'); if (location.city) { - datasetLocationQb.addSelect( - `coalesce(similarity(city, '${location.city}'), 0)`, - 'similarityCity', - ); + datasetLocationQb + .setParameter('locationCity', location.city) + .addSelect( + `coalesce(similarity(city, :locationCity), 0)`, + 'similarityCity', + ); datasetLocationQb.addOrderBy('"similarityCity"', 'DESC'); } if (location.subdivision) { - datasetLocationQb.addSelect( - `coalesce(similarity(subdivision, '${location.subdivision}'), 0)`, - 'similaritySubdivision', - ); + datasetLocationQb + .setParameter('locationSubdivision', location.subdivision) + .addSelect( + `coalesce(similarity(subdivision, :locationSubdivision), 0)`, + 'similaritySubdivision', + ); datasetLocationQb.addOrderBy('"similaritySubdivision"', 'DESC'); } if (location.country) { - datasetLocationQb.addSelect( - `coalesce(similarity(country, '${location.country}'), 0)`, - 'similarityCountry', - ); + datasetLocationQb + .setParameter('locationCountry', location.country) + .addSelect( + `coalesce(similarity(country, :locationCountry), 0)`, + 'similarityCountry', + ); datasetLocationQb.addOrderBy('"similarityCountry"', 'DESC'); } From 2dfedd56777dd82fabac69a0c054f6f7bcfe9e4d Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 17:05:13 +0100 Subject: [PATCH 06/14] feat: add query to query errors --- bin/importProfileFromJSON.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index d16ba38df0..ab1ba676d9 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -102,6 +102,7 @@ const main = async () => { console.error({ type: 'db_query_failed', message: error.message, + query: error.query, filePath, }); } else if (error instanceof z.ZodError) { From 3db28abbd64f99f7cd928e60492a2936ef543789 Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 17:06:52 +0100 Subject: [PATCH 07/14] feat: adjust docs and remove test userid --- bin/importProfileFromJSON.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index ab1ba676d9..2053cdf65c 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -12,7 +12,13 @@ import { randomUUID } from 'node:crypto'; /** * Import profile from JSON to user by id * - * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/testuser.json -u testuser + * Single file usage: + * + * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/testuser.json + * + * Directory usage: + * + * npx ts-node bin/importProfileFromJSON.ts --path ~/Downloads/profiles --limit 100 --offset 0 --import import_run_test */ const main = async () => { let con: DataSource | null = null; @@ -92,7 +98,7 @@ const main = async () => { await importUserExperienceFromJSON({ con: con.manager, dataJson: dataJSON, - userId: 'testuser', + userId, importId: params.uid, }); } catch (error) { From eb8779542fd29ff1fda735152095d7289108da53 Mon Sep 17 00:00:00 2001 From: capJavert Date: Mon, 24 Nov 2025 17:12:44 +0100 Subject: [PATCH 08/14] feat: adjust logging --- bin/importProfileFromJSON.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index 2053cdf65c..00e54d8917 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -54,7 +54,7 @@ const main = async () => { const params = paramsSchema.parse(values); - console.log('Starting import with ID:', params.uid); + console.log(`Starting import with ID: ${params.uid}`); con = await createOrGetConnection(); @@ -68,12 +68,10 @@ const main = async () => { filePaths.sort(); // ensure consistent order for offset/limit - console.log('Found files:', filePaths.length); + console.log(`Found files: ${filePaths.length}`); console.log( - `Importing:`, - Math.min(params.limit, filePaths.length), - `(limit ${params.limit})`, + `Importing: ${Math.min(params.limit, filePaths.length)} (limit ${params.limit}, offset ${params.offset})`, ); for (const [index, fileName] of filePaths @@ -124,7 +122,7 @@ const main = async () => { } if (index && index % 100 === 0) { - console.log('Done so far:', index, ', failed:', failedImports); + console.log(`Done so far: ${index}, failed: ${failedImports}`); } } } catch (error) { From 5f6a965328d2a3383ba9b42a6ae4642c4d647c4d Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 11:07:21 +0100 Subject: [PATCH 09/14] feat: support employment type --- src/common/profile/import.ts | 13 ++++++++++--- src/common/schema/profile.ts | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index e417fdfe87..fd437b5b3b 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -11,7 +11,7 @@ import { Company } from '../../../src/entity/Company'; import { UserExperienceWork } from '../../../src/entity/user/experiences/UserExperienceWork'; import { insertOrIgnoreUserExperienceSkills } from '../../../src/entity/user/experiences/UserExperienceSkill'; import { textFromEnumValue } from '../../../src/common'; -import { LocationType } from '@dailydotdev/schema'; +import { EmploymentType, LocationType } from '@dailydotdev/schema'; import { DatasetLocation } from '../../../src/entity/dataset/DatasetLocation'; import { UserExperienceEducation } from '../../../src/entity/user/experiences/UserExperienceEducation'; import { UserExperienceCertification } from '../../../src/entity/user/experiences/UserExperienceCertification'; @@ -149,6 +149,7 @@ export const importUserExperienceWork = async ({ ended_at: endedAt, location, flags, + employment_type: employmentType, } = userExperience; const insertResult = await con.getRepository(UserExperienceWork).insert( @@ -166,7 +167,6 @@ export const importUserExperienceWork = async ({ locationType: locationType ? (Object.entries(LocationType).find(([, value]) => { return ( - // TODO cv-parsing remove this replace when cv is adjusted to not use prefix locationType.replace('LOCATION_TYPE_', '') === textFromEnumValue(LocationType, value) ); @@ -176,6 +176,14 @@ export const importUserExperienceWork = async ({ location, con: con, })), + employmentType: employmentType + ? (Object.entries(EmploymentType).find(([, value]) => { + return ( + employmentType.replace('EMPLOYMENT_TYPE_', '') === + textFromEnumValue(EmploymentType, value) + ); + })?.[1] as EmploymentType) + : undefined, }), ); @@ -201,7 +209,6 @@ export const importUserExperienceEducation = async ({ }): Promise<{ experienceId: string }> => { const userExperience = userExperienceEducationImportSchema.parse(data); - // TODO cv-parsing potentially won't be needed once cv is adjusted to use camelCase const { company, title, diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index 04dafc67f7..6689ca44b8 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -122,6 +122,7 @@ export const userExperienceWorkImportSchema = z.object({ }) .nullish(), flags: z.object({ import: z.string() }).partial().optional(), + employment_type: z.string().nullish(), }); export const userExperienceEducationImportSchema = z.object({ From de90fee4333164904f851472d574193c0ff583cf Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 11:30:08 +0100 Subject: [PATCH 10/14] feat: support url parsing --- src/common/profile/import.ts | 4 ++++ src/common/schema/common.ts | 18 ++++++++++++++++++ src/common/schema/profile.ts | 4 +++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index fd437b5b3b..8e7c0329df 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -269,6 +269,7 @@ export const importUserExperienceCertification = async ({ started_at: startedAt, ended_at: endedAt, flags, + url, } = userExperience; const insertResult = await con @@ -284,6 +285,7 @@ export const importUserExperienceCertification = async ({ title, startedAt, endedAt, + url, }), ); @@ -312,6 +314,7 @@ export const importUserExperienceProject = async ({ ended_at: endedAt, skills, flags, + url, } = userExperience; const insertResult = await con.getRepository(UserExperienceProject).insert( @@ -322,6 +325,7 @@ export const importUserExperienceProject = async ({ description, startedAt, endedAt, + url, }), ); diff --git a/src/common/schema/common.ts b/src/common/schema/common.ts index 84386c1b54..af592d99fd 100644 --- a/src/common/schema/common.ts +++ b/src/common/schema/common.ts @@ -16,3 +16,21 @@ export const paginationSchema = z.object({ }); export type PaginationArgs = z.infer; + +const urlStartRegexMatch = /^https?:\/\//; + +// match http(s) urls and partials like daily.dev (without protocol ) +export const urlParseSchema = z.preprocess( + (val) => { + if (typeof val === 'string') { + return val.match(urlStartRegexMatch) ? val : `https://${val}`; + } + + return val; + }, + z.url({ + protocol: /^https?$/, + hostname: z.regexes.domain, + normalize: true, + }), +); diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index 6689ca44b8..c5a4f4543a 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -1,6 +1,6 @@ import z from 'zod'; import { UserExperienceType } from '../../entity/user/experiences/types'; -import { paginationSchema } from './common'; +import { paginationSchema, urlParseSchema } from './common'; export const userExperiencesSchema = z .object({ @@ -161,6 +161,7 @@ export const userExperienceCertificationImportSchema = z.object({ started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), flags: z.object({ import: z.string() }).partial().optional(), + url: urlParseSchema.nullish(), }); export const userExperienceProjectImportSchema = z.object({ @@ -178,4 +179,5 @@ export const userExperienceProjectImportSchema = z.object({ .nullish() .transform((n) => (n === null ? undefined : n)), flags: z.object({ import: z.string() }).partial().optional(), + url: urlParseSchema.nullish(), }); From 550edb8c387f2f482481de343e7fe0884ad91c9a Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 11:31:08 +0100 Subject: [PATCH 11/14] feat: support grade --- src/common/profile/import.ts | 2 ++ src/common/schema/profile.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index 8e7c0329df..b5416147e1 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -219,6 +219,7 @@ export const importUserExperienceEducation = async ({ location, subtitle, flags, + grade, } = userExperience; const insertResult = await con.getRepository(UserExperienceEducation).insert( @@ -238,6 +239,7 @@ export const importUserExperienceEducation = async ({ con: con, })), subtitle, + grade, }), ); diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index c5a4f4543a..efff22e220 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -148,6 +148,7 @@ export const userExperienceEducationImportSchema = z.object({ .transform((n) => (n === null ? undefined : n)), subtitle: z.string().nullish(), flags: z.object({ import: z.string() }).partial().optional(), + grade: z.string().nullish(), }); export const userExperienceCertificationImportSchema = z.object({ From 53cbc90514935ce40004b707e929594ddb960049 Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 11:59:42 +0100 Subject: [PATCH 12/14] feat: add transaction support from outside --- bin/importProfileFromJSON.ts | 100 ++++++++++++---------- src/common/profile/import.ts | 10 ++- src/workers/opportunity/parseCVProfile.ts | 1 + 3 files changed, 63 insertions(+), 48 deletions(-) diff --git a/bin/importProfileFromJSON.ts b/bin/importProfileFromJSON.ts index 00e54d8917..b705eb92aa 100644 --- a/bin/importProfileFromJSON.ts +++ b/bin/importProfileFromJSON.ts @@ -74,57 +74,61 @@ const main = async () => { `Importing: ${Math.min(params.limit, filePaths.length)} (limit ${params.limit}, offset ${params.offset})`, ); - for (const [index, fileName] of filePaths - .slice(params.offset, params.offset + params.limit) - .entries()) { - const filePath = - params.path === fileName ? fileName : path.join(params.path, fileName); - - try { - if (!filePath.endsWith('.json')) { - throw { type: 'not_json_ext', filePath }; - } - - const userId = filePath.split('/').pop()?.split('.json')[0]; - - if (!userId) { - throw { type: 'no_user_id', filePath }; - } - - const dataJSON = JSON.parse(await readFile(filePath, 'utf-8')); - - await importUserExperienceFromJSON({ - con: con.manager, - dataJson: dataJSON, - userId, - importId: params.uid, - }); - } catch (error) { - failedImports += 1; - - if (error instanceof QueryFailedError) { - console.error({ - type: 'db_query_failed', - message: error.message, - query: error.query, - filePath, + await con.transaction(async (entityManager) => { + for (const [index, fileName] of filePaths + .slice(params.offset, params.offset + params.limit) + .entries()) { + const filePath = + params.path === fileName + ? fileName + : path.join(params.path, fileName); + + try { + if (!filePath.endsWith('.json')) { + throw { type: 'not_json_ext', filePath }; + } + + const userId = filePath.split('/').pop()?.split('.json')[0]; + + if (!userId) { + throw { type: 'no_user_id', filePath }; + } + + const dataJSON = JSON.parse(await readFile(filePath, 'utf-8')); + + await importUserExperienceFromJSON({ + con: entityManager, + dataJson: dataJSON, + userId, + importId: params.uid, }); - } else if (error instanceof z.ZodError) { - console.error({ - type: 'zod_error', - message: error.issues[0].message, - path: error.issues[0].path, - filePath, - }); - } else { - console.error(error); + } catch (error) { + failedImports += 1; + + if (error instanceof QueryFailedError) { + console.error({ + type: 'db_query_failed', + message: error.message, + query: error.query, + filePath, + }); + } else if (error instanceof z.ZodError) { + console.error({ + type: 'zod_error', + message: error.issues[0].message, + path: error.issues[0].path, + filePath, + }); + } else { + console.error(error); + } } - } - if (index && index % 100 === 0) { - console.log(`Done so far: ${index}, failed: ${failedImports}`); + if (index && index % 100 === 0) { + console.log(`Done so far: ${index}, failed: ${failedImports}`); + } } - } + }); } catch (error) { console.error(error instanceof z.ZodError ? z.prettifyError(error) : error); } finally { @@ -134,6 +138,8 @@ const main = async () => { if (failedImports > 0) { console.log(`Failed imports: ${failedImports}`); + } else { + console.log('Done!'); } process.exit(0); diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index b5416147e1..bf39d0f02a 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -347,11 +347,13 @@ export const importUserExperienceFromJSON = async ({ dataJson, userId, importId, + transaction = false, }: { con: EntityManager; dataJson: unknown; userId: string; importId?: string; + transaction?: boolean; }) => { if (!userId) { throw new Error('userId is required'); @@ -380,7 +382,13 @@ export const importUserExperienceFromJSON = async ({ ) .parse(dataJson); - await con.transaction(async (entityManager) => { + const transactionFn = transaction + ? con.transaction + : async (callback: (entityManager: EntityManager) => Promise) => { + return callback(con); + }; + + await transactionFn(async (entityManager) => { for (const item of data) { const importData = { ...item, diff --git a/src/workers/opportunity/parseCVProfile.ts b/src/workers/opportunity/parseCVProfile.ts index 2986360225..f13b75601e 100644 --- a/src/workers/opportunity/parseCVProfile.ts +++ b/src/workers/opportunity/parseCVProfile.ts @@ -92,6 +92,7 @@ export const parseCVProfileWorker: TypedWorker<'api.v1.candidate-preference-upda con: con.manager, dataJson, userId, + transaction: true, }); } catch (error) { // revert to previous date on error From 850410c6fdd6da9dbc0d669167851cc47f564914 Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 11:59:54 +0100 Subject: [PATCH 13/14] feat: catch invalid urls --- src/common/schema/profile.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/schema/profile.ts b/src/common/schema/profile.ts index efff22e220..613426a3e2 100644 --- a/src/common/schema/profile.ts +++ b/src/common/schema/profile.ts @@ -162,7 +162,7 @@ export const userExperienceCertificationImportSchema = z.object({ started_at: z.coerce.date().default(() => new Date()), ended_at: z.coerce.date().nullish().default(null), flags: z.object({ import: z.string() }).partial().optional(), - url: urlParseSchema.nullish(), + url: urlParseSchema.nullish().catch(undefined), }); export const userExperienceProjectImportSchema = z.object({ @@ -180,5 +180,5 @@ export const userExperienceProjectImportSchema = z.object({ .nullish() .transform((n) => (n === null ? undefined : n)), flags: z.object({ import: z.string() }).partial().optional(), - url: urlParseSchema.nullish(), + url: urlParseSchema.nullish().catch(undefined), }); From b534305588ce49f81b92c63ead56d99349cf37fb Mon Sep 17 00:00:00 2001 From: capJavert Date: Tue, 25 Nov 2025 12:11:59 +0100 Subject: [PATCH 14/14] fix: transaction --- src/common/profile/import.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/profile/import.ts b/src/common/profile/import.ts index bf39d0f02a..60086eb7bc 100644 --- a/src/common/profile/import.ts +++ b/src/common/profile/import.ts @@ -382,11 +382,11 @@ export const importUserExperienceFromJSON = async ({ ) .parse(dataJson); - const transactionFn = transaction - ? con.transaction - : async (callback: (entityManager: EntityManager) => Promise) => { - return callback(con); - }; + const transactionFn = async ( + callback: (entityManager: EntityManager) => Promise, + ) => { + return transaction ? con.transaction(callback) : callback(con); + }; await transactionFn(async (entityManager) => { for (const item of data) {