-
-
Notifications
You must be signed in to change notification settings - Fork 82
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Innei <i@innei.in>
- Loading branch information
Showing
18 changed files
with
436 additions
and
288 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
262 changes: 262 additions & 0 deletions
262
apps/core/src/modules/ai/ai-summary/ai-summary.service.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,262 @@ | ||
| import removeMdCodeblock from 'remove-md-codeblock' | ||
|
|
||
| import { Injectable, Logger } from '@nestjs/common' | ||
| import { OnEvent } from '@nestjs/event-emitter' | ||
|
|
||
| import { BizException } from '~/common/exceptions/biz.exception' | ||
| import { BusinessEvents } from '~/constants/business-event.constant' | ||
| import { CollectionRefTypes } from '~/constants/db.constant' | ||
| import { ErrorCodeEnum } from '~/constants/error-code.constant' | ||
| import { DatabaseService } from '~/processors/database/database.service' | ||
| import { CacheService } from '~/processors/redis/cache.service' | ||
| import { InjectModel } from '~/transformers/model.transformer' | ||
| import { transformDataToPaginate } from '~/transformers/paginate.transformer' | ||
| import { md5 } from '~/utils' | ||
|
|
||
| import { ConfigsService } from '../../configs/configs.service' | ||
| import { DEFAULT_SUMMARY_LANG, LANGUAGE_CODE_TO_NAME } from '../ai.constants' | ||
| import { AiService } from '../ai.service' | ||
| import { AISummaryModel } from './ai-summary.model' | ||
| import type { PagerDto } from '~/shared/dto/pager.dto' | ||
|
|
||
| @Injectable() | ||
| export class AiSummaryService { | ||
| private readonly logger: Logger | ||
| constructor( | ||
| @InjectModel(AISummaryModel) | ||
| private readonly aiSummaryModel: MongooseModel<AISummaryModel>, | ||
| private readonly databaseService: DatabaseService, | ||
| private readonly configService: ConfigsService, | ||
|
|
||
| private readonly cacheService: CacheService, | ||
| private readonly aiService: AiService, | ||
| ) { | ||
| this.logger = new Logger(AiSummaryService.name) | ||
| } | ||
|
|
||
| private cachedTaskId2AiPromise = new Map<string, Promise<any>>() | ||
|
|
||
| private serializeText(text: string) { | ||
| return removeMdCodeblock(text) | ||
| } | ||
| async generateSummaryByOpenAI( | ||
| articleId: string, | ||
| lang = DEFAULT_SUMMARY_LANG, | ||
| ) { | ||
| const { | ||
| ai: { enableSummary, openAiPreferredModel }, | ||
| } = await this.configService.waitForConfigReady() | ||
|
|
||
| if (!enableSummary) { | ||
| throw new BizException(ErrorCodeEnum.AINotEnabled) | ||
| } | ||
|
|
||
| const openai = await this.aiService.getOpenAiClient() | ||
|
|
||
| const article = await this.databaseService.findGlobalById(articleId) | ||
| if (!article) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess) | ||
| } | ||
|
|
||
| if (article.type === CollectionRefTypes.Recently) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess) | ||
| } | ||
|
|
||
| const taskId = `ai:summary:${articleId}:${lang}` | ||
| try { | ||
| if (this.cachedTaskId2AiPromise.has(taskId)) { | ||
| return this.cachedTaskId2AiPromise.get(taskId) | ||
| } | ||
| const redis = this.cacheService.getClient() | ||
|
|
||
| const isProcessing = await redis.get(taskId) | ||
|
|
||
| if (isProcessing === 'processing') { | ||
| throw new BizException(ErrorCodeEnum.AIProcessing) | ||
| } | ||
|
|
||
| const taskPromise = handle.bind(this)( | ||
| articleId, | ||
| this.serializeText(article.document.text), | ||
| article.document.title, | ||
| ) as Promise<any> | ||
|
|
||
| this.cachedTaskId2AiPromise.set(taskId, taskPromise) | ||
| return await taskPromise | ||
|
|
||
| async function handle( | ||
| this: AiSummaryService, | ||
| id: string, | ||
| text: string, | ||
| title: string, | ||
| ) { | ||
| // 等待 30s | ||
| await redis.set(taskId, 'processing', 'EX', 30) | ||
|
|
||
| const completion = await openai.chat.completions.create({ | ||
| messages: [ | ||
| { | ||
| role: 'user', | ||
| content: `Summarize this article in ${LANGUAGE_CODE_TO_NAME[lang] || 'Chinese'} to 150 words: | ||
| "${text}" | ||
| CONCISE SUMMARY:`, | ||
| }, | ||
| ], | ||
| model: openAiPreferredModel, | ||
| }) | ||
|
|
||
| await redis.del(taskId) | ||
|
|
||
| const summary = completion.choices[0].message.content | ||
|
|
||
| this.logger.log( | ||
| `OpenAI 生成文章 ${id} 「${title}」的摘要花费了 ${completion.usage?.total_tokens}token`, | ||
| ) | ||
| const contentMd5 = md5(text) | ||
|
|
||
| const doc = await this.aiSummaryModel.create({ | ||
| hash: contentMd5, | ||
| lang, | ||
| refId: id, | ||
| summary, | ||
| }) | ||
|
|
||
| return doc | ||
| } | ||
| } catch (error) { | ||
| this.logger.error( | ||
| `OpenAI 在处理文章 ${articleId} 时出错：${error.message}`, | ||
| ) | ||
|
|
||
| throw new BizException(ErrorCodeEnum.AIException, error.message) | ||
| } finally { | ||
| this.cachedTaskId2AiPromise.delete(taskId) | ||
| } | ||
| } | ||
|
|
||
| async getSummariesByRefId(refId: string) { | ||
| const article = await this.databaseService.findGlobalById(refId) | ||
|
|
||
| if (!article) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFound) | ||
| } | ||
| const summaries = await this.aiSummaryModel.find({ | ||
| refId, | ||
| }) | ||
|
|
||
| return { | ||
| summaries, | ||
| article, | ||
| } | ||
| } | ||
|
|
||
| async getAllSummaries(pager: PagerDto) { | ||
| const { page, size } = pager | ||
| const summaries = await this.aiSummaryModel.paginate( | ||
| {}, | ||
| { | ||
| page, | ||
| limit: size, | ||
| sort: { | ||
| created: -1, | ||
| }, | ||
| lean: true, | ||
| leanWithId: true, | ||
| }, | ||
| ) | ||
| const data = transformDataToPaginate(summaries) | ||
|
|
||
| return { | ||
| ...data, | ||
| articles: await this.getRefArticles(summaries.docs), | ||
| } | ||
| } | ||
|
|
||
| private getRefArticles(docs: AISummaryModel[]) { | ||
| return this.databaseService | ||
| .findGlobalByIds(docs.map((d) => d.refId)) | ||
| .then((articles) => { | ||
| const articleMap = {} as Record< | ||
| string, | ||
| { title: string; id: string; type: CollectionRefTypes } | ||
| > | ||
| for (const a of articles.notes) { | ||
| articleMap[a.id] = { | ||
| title: a.title, | ||
| id: a.id, | ||
| type: CollectionRefTypes.Note, | ||
| } | ||
| } | ||
|
|
||
| for (const a of articles.posts) { | ||
| articleMap[a.id] = { | ||
| title: a.title, | ||
| id: a.id, | ||
| type: CollectionRefTypes.Post, | ||
| } | ||
| } | ||
| return articleMap | ||
| }) | ||
| } | ||
|
|
||
| async updateSummaryInDb(id: string, summary: string) { | ||
| const doc = await this.aiSummaryModel.findById(id) | ||
| if (!doc) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess) | ||
| } | ||
|
|
||
| doc.summary = summary | ||
| await doc.save() | ||
| return doc | ||
| } | ||
| async getSummaryByArticleId(articleId: string, lang = DEFAULT_SUMMARY_LANG) { | ||
| const article = await this.databaseService.findGlobalById(articleId) | ||
| if (!article) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess) | ||
| } | ||
|
|
||
| if (article.type === CollectionRefTypes.Recently) { | ||
| throw new BizException(ErrorCodeEnum.ContentNotFoundCantProcess) | ||
| } | ||
|
|
||
| const contentMd5 = md5(this.serializeText(article.document.text)) | ||
| const doc = await this.aiSummaryModel.findOne({ | ||
| hash: contentMd5, | ||
|
|
||
| lang, | ||
| }) | ||
|
|
||
| return doc | ||
| } | ||
|
|
||
| async deleteSummaryByArticleId(articleId: string) { | ||
| await this.aiSummaryModel.deleteMany({ | ||
| refId: articleId, | ||
| }) | ||
| } | ||
|
|
||
| async deleteSummaryInDb(id: string) { | ||
| await this.aiSummaryModel.deleteOne({ | ||
| _id: id, | ||
| }) | ||
| } | ||
|
|
||
| @OnEvent(BusinessEvents.POST_DELETE) | ||
| @OnEvent(BusinessEvents.NOTE_DELETE) | ||
| async handleDeleteArticle(event: { id: string }) { | ||
| await this.deleteSummaryByArticleId(event.id) | ||
| } | ||
|
|
||
| @OnEvent(BusinessEvents.POST_CREATE) | ||
| @OnEvent(BusinessEvents.NOTE_CREATE) | ||
| async handleCreateArticle(event: { id: string }) { | ||
| const enableAutoGenerate = await this.configService | ||
| .get('ai') | ||
| .then((c) => c.enableAutoGenerateSummary && c.enableSummary) | ||
| if (!enableAutoGenerate) { | ||
| return | ||
| } | ||
| await this.generateSummaryByOpenAI(event.id) | ||
| } | ||
| } |
19 changes: 19 additions & 0 deletions
19
apps/core/src/modules/ai/ai-writer/ai-writer.controller.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| import { Body, Post } from '@nestjs/common' | ||
| import { ApiController } from '~/common/decorators/api-controller.decorator' | ||
| import { AiQueryType, GenerateAiDto } from './ai-writer.dto' | ||
| import { AiWriterService } from './ai-writer.service' | ||
|
|
||
| @ApiController('ai/writer') | ||
| export class AiWriterController { | ||
| constructor(private readonly aiWriterService: AiWriterService) {} | ||
|
|
||
| @Post('generate') | ||
| async generate(@Body() body: GenerateAiDto) { | ||
| switch (body.type) { | ||
| case AiQueryType.TitleSlug: | ||
| return this.aiWriterService.generateTitleAndSlugByOpenAI(body.text) | ||
| case AiQueryType.Title: | ||
| return this.aiWriterService.generateSlugByTitleViaOpenAI(body.title) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| import { IsEnum, IsString, ValidateIf } from 'class-validator' | ||
|
|
||
| export enum AiQueryType { | ||
| TitleSlug = 'title-slug', | ||
| Title = 'title', | ||
| } | ||
|
|
||
| export class GenerateAiDto { | ||
| @IsEnum(AiQueryType) | ||
| type: AiQueryType | ||
|
|
||
| @ValidateIf((o: GenerateAiDto) => o.type === AiQueryType.TitleSlug) | ||
| @IsString() | ||
| text: string | ||
| @ValidateIf((o: GenerateAiDto) => o.type === AiQueryType.Title) | ||
| @IsString() | ||
| title: string | ||
| } |
Oops, something went wrong.