Skip to content

Commit

Permalink
fix(server): truncate embedding tables (#7449)
Browse files Browse the repository at this point in the history
truncate
  • Loading branch information
mertalev authored Feb 27, 2024
1 parent 9fa2424 commit fb18129
Show file tree
Hide file tree
Showing 6 changed files with 16 additions and 16 deletions.
1 change: 1 addition & 0 deletions server/src/domain/repositories/search.repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,4 +187,5 @@ export interface ISearchRepository {
searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;
deleteAllSearchEmbeddings(): Promise<void>;
}
2 changes: 2 additions & 0 deletions server/src/domain/smart-info/smart-info.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ describe(SmartInfoService.name, () => {

expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.SMART_SEARCH, data: { id: assetStub.image.id } }]);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.SMART_SEARCH);
expect(searchMock.deleteAllSearchEmbeddings).not.toHaveBeenCalled();
});

it('should queue all the assets', async () => {
Expand All @@ -83,6 +84,7 @@ describe(SmartInfoService.name, () => {

expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.SMART_SEARCH, data: { id: assetStub.image.id } }]);
expect(assetMock.getAll).toHaveBeenCalled();
expect(searchMock.deleteAllSearchEmbeddings).toHaveBeenCalled();
});
});

Expand Down
4 changes: 4 additions & 0 deletions server/src/domain/smart-info/smart-info.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ export class SmartInfoService {
return true;
}

if (force) {
await this.repository.deleteAllSearchEmbeddings();
}

const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
Expand Down
4 changes: 2 additions & 2 deletions server/src/infra/repositories/person.repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,11 @@ export class PersonRepository implements IPersonRepository {
}

async deleteAll(): Promise<void> {
await this.personRepository.delete({});
await this.personRepository.clear();
}

async deleteAllFaces(): Promise<void> {
await this.assetFaceRepository.delete({});
await this.assetFaceRepository.query('TRUNCATE TABLE asset_faces CASCADE');
}

getAllFaces(
Expand Down
20 changes: 6 additions & 14 deletions server/src/infra/repositories/search.repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,25 +229,17 @@ export class SearchRepository implements ISearchRepository {
this.logger.log(`Updating database CLIP dimension size to ${dimSize}.`);

await this.smartSearchRepository.manager.transaction(async (manager) => {
await manager.query(`DROP TABLE smart_search`);

await manager.query(`
CREATE TABLE smart_search (
"assetId" uuid PRIMARY KEY REFERENCES assets(id) ON DELETE CASCADE,
embedding vector(${dimSize}) NOT NULL )`);

await manager.query(`
CREATE INDEX clip_index ON smart_search
USING vectors (embedding vector_cos_ops) WITH (options = $$
[indexing.hnsw]
m = 16
ef_construction = 300
$$)`);
await manager.clear(SmartSearchEntity);
await manager.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET DATA TYPE vector(${dimSize})`);
});

this.logger.log(`Successfully updated database CLIP dimension size from ${curDimSize} to ${dimSize}.`);
}

deleteAllSearchEmbeddings(): Promise<void> {
return this.smartSearchRepository.clear();
}

private async getDimSize(): Promise<number> {
const res = await this.smartSearchRepository.manager.query(`
SELECT atttypmod as dimsize
Expand Down
1 change: 1 addition & 0 deletions server/test/repositories/search.repository.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ export const newSearchRepositoryMock = (): jest.Mocked<ISearchRepository> => {
searchFaces: jest.fn(),
upsert: jest.fn(),
searchPlaces: jest.fn(),
deleteAllSearchEmbeddings: jest.fn(),
};
};

0 comments on commit fb18129

Please sign in to comment.