Skip to content

Commit

Permalink
支持上传文档或图像并解读
Browse files Browse the repository at this point in the history
  • Loading branch information
@Vinlic
Vinlic committed Mar 31, 2024
1 parent dc1481f commit 19ecdcf
Show file tree
Hide file tree
Showing 4 changed files with 276 additions and 15 deletions.
122 changes: 120 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,11 +239,129 @@ Authorization: Bearer [refresh_token]

### 文档解读

接口开发中...
提供一个可访问的文件URL或者BASE64_URL进行解析。

**POST /v1/chat/completions**

header 需要设置 Authorization 头部：

```
Authorization: Bearer [refresh_token]
```

请求数据：
```json
{
// 模型名称随意填写，如果不希望输出检索过程模型名称请包含silent_search
"model": "kimi",
"messages": [
{
"role": "user",
"content": [
{
"type": "file",
"file_url": {
"url": "https://mj101-1317487292.cos.ap-shanghai.myqcloud.com/ai/test.pdf"
}
},
{
"type": "text",
"text": "文档里说了什么？"
}
]
}
]
}
```

响应数据：
```json
{
"id": "85774360661086208",
"model": "step",
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "这是一个关于爱情魔法的文档。它包含了四个部分：\n\n1. **PMG 4.1390 – 1495**：这是一个使用面包和咒语来吸引心仪女性的仪式。仪式中需要将面包分成七个小块，并在特定地点进行咒语的念诵和投掷。\n2. **PMG 4.1342 – 57**：这是一个召唤恶魔来使一个名叫Tereous的女性受到折磨，直到她与一个名叫Didymos的人相爱并结合的咒语。\n3. **PGM 4.1265 – 74**：这是关于如何赢得一个美丽的女人的咒语。它涉及到连续三天保持纯洁，向女神阿佛洛狄特（Aphrodite）供奉乳香，并在心中默念她的神秘名字。\n4. **PGM 4.1496 – 1**：这是一个使用没药来吸引一个特定女性的咒语。这个咒语需要在煤上焚烧没药的同时念诵，目的是让这个女性心中只想着施咒者，并最终与施咒者相爱。"
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2
},
"created": 1711903489
}
```

### 图像解析

接口开发中...
提供一个可访问的图像URL或者BASE64_URL进行解析。

此格式兼容 [gpt-4-vision-preview](https://platform.openai.com/docs/guides/vision) API格式，您也可以用这个格式传送文档进行解析。

**POST /v1/chat/completions**

header 需要设置 Authorization 头部：

```
Authorization: Bearer [refresh_token]
```

请求数据：
```json
{
// 模型名称随意填写
"model": "step",
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://k.sinaimg.cn/n/sinakd20111/106/w1024h682/20240327/babd-2ce15fdcfbd6ddbdc5ab588c29b3d3d9.jpg/w700d1q75cms.jpg"
}
},
{
"type": "text",
"text": "图像描述了什么？"
}
]
}
]
}
```

响应数据：
```json
{
"id": "85773574417829888",
"model": "step",
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "这张图片展示了一个活动现场，似乎是某种新产品或技术的发布会。图片中央有一个大屏幕，上面写着“创新技术及产品首发”，屏幕上还展示了一些公司的标志或名称，如“RWKV”、“财跃星辰”、“阶跃星辰”、“商汤”和“零方科技”。在屏幕下方的舞台上，有几位穿着正装的人士正在进行互动，可能是在进行产品发布或演示。整个场景给人一种正式且科技感十足的印象。"
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2
},
"created": 1711903302
}
```

## 注意事项

Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "step-free-api",
"version": "0.0.4",
"version": "0.0.5",
"description": "Stepchat Free API Server",
"type": "module",
"main": "dist/index.js",
Expand Down
4 changes: 3 additions & 1 deletion src/api/consts/exceptions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,7 @@ export default {
API_TOKEN_EXPIRES: [-2002, 'Token已失效'],
API_FILE_URL_INVALID: [-2003, '远程文件URL非法'],
API_FILE_EXECEEDS_SIZE: [-2004, '远程文件超出大小'],
API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出']
API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出'],
API_FILE_UPLOAD_FAILED: [-2006, '文件上传失败'],
API_FILE_UPLOAD_TIMEOUT: [-2007, '文件上传超时']
}
163 changes: 152 additions & 11 deletions src/api/controllers/chat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,12 @@ async function createCompletion(
logger.info(messages);

// 提取引用文件URL并上传step获得引用的文件ID列表
// const refFileUrls = extractRefFileUrls(messages);
// const refs = refFileUrls.length ? await Promise.all(refFileUrls.map(fileUrl => uploadFile(fileUrl, refreshToken))) : [];
const refFileUrls = extractRefFileUrls(messages);
const refs = refFileUrls.length
? await Promise.all(
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
: [];

// 创建会话
const convId = await createConversation("新会话", refreshToken);
Expand All @@ -217,7 +221,7 @@ async function createCompletion(
const { deviceId, token } = await acquireToken(refreshToken);
const result = await axios.post(
`https://stepchat.cn/api/proto.chat.v1.ChatMessageService/SendMessageStream`,
messagesPrepare(convId, messages),
messagesPrepare(convId, messages, refs),
{
headers: {
"Content-Type": "application/connect+json",
Expand Down Expand Up @@ -283,12 +287,12 @@ async function createCompletionStream(
logger.info(messages);

// 提取引用文件URL并上传step获得引用的文件ID列表
// const refFileUrls = extractRefFileUrls(messages);
// const refs = refFileUrls.length
// ? await Promise.all(
// refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
// )
// : [];
const refFileUrls = extractRefFileUrls(messages);
const refs = refFileUrls.length
? await Promise.all(
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
: [];

// 创建会话
const convId = await createConversation("新会话", refreshToken);
Expand All @@ -297,7 +301,7 @@ async function createCompletionStream(
const { deviceId, token } = await acquireToken(refreshToken);
const result = await axios.post(
`https://stepchat.cn/api/proto.chat.v1.ChatMessageService/SendMessageStream`,
messagesPrepare(convId, messages),
messagesPrepare(convId, messages, refs),
{
headers: {
"Content-Type": "application/connect+json",
Expand Down Expand Up @@ -384,7 +388,7 @@ function extractRefFileUrls(messages: any[]) {
*
* @param messages 参考gpt系列消息格式，多轮对话请完整提供上下文
*/
function messagesPrepare(convId: string, messages: any[]) {
function messagesPrepare(convId: string, messages: any[], refs: any[]) {
const content = messages.reduce((content, message) => {
if (_.isArray(message.content)) {
return message.content.reduce((_content, v) => {
Expand All @@ -398,6 +402,7 @@ function messagesPrepare(convId: string, messages: any[]) {
chatId: convId,
messageInfo: {
text: content,
attachments: refs.length > 0 ? refs : undefined,
},
});
const data = wrapData(json);
Expand Down Expand Up @@ -698,6 +703,142 @@ function generateCookie(deviceId: string, accessToken: string) {
return [`Oasis-Token=${accessToken}`, `Oasis-Webid=${deviceId}`].join("; ");
}

/**
* 预检查文件URL有效性
*
* @param fileUrl 文件URL
*/
async function checkFileUrl(fileUrl: string) {
if (util.isBASE64Data(fileUrl)) return;
const result = await axios.head(fileUrl, {
timeout: 15000,
headers: {
Cookie:
"INGRESSCOOKIE=1711735363.098.26095.391795|cdfd1cd25bff0dd747986e2907e40a4e; Oasis-Webid=9b4315f03ff3e2abdf7a0b6eb1d41b293ac6fa12; Oasis-Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY3RpdmF0ZWQiOmZhbHNlLCJhZ2UiOjgsImJhbmVkIjpmYWxzZSwiZXhwIjoxNzExOTAzODkxLCJtb2RlIjoyLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.KrkngKk8drUVfgRBnEE3A07JKjmqgHL3c7J5PlMxKWw...eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBfaWQiOjEwMzAwLCJkZXZpY2VfaWQiOiI5YjQzMTVmMDNmZjNlMmFiZGY3YTBiNmViMWQ0MWIyOTNhYzZmYTEyIiwiZXhwIjoxNzEzMDMxMzkxLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.HRfkpUOFNGO0Jm6wvijGg9PzxD9d9-j4gXh4eqOkAKk",
Referer: "https://platform.stepfun.com/",
UserAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
},
validateStatus: () => true,
});
if (result.status >= 400)
throw new APIException(
EX.API_FILE_URL_INVALID,
`File ${fileUrl} is not valid: [${result.status}] ${result.statusText}`
);
// 检查文件大小
if (result.headers && result.headers["content-length"]) {
const fileSize = parseInt(result.headers["content-length"], 10);
if (fileSize > FILE_MAX_SIZE)
throw new APIException(
EX.API_FILE_EXECEEDS_SIZE,
`File ${fileUrl} is not valid`
);
}
}

/**
* 上传文件
*
* @param fileUrl 文件URL
* @param refreshToken 用于刷新access_token的refresh_token
*/
async function uploadFile(fileUrl: string, refreshToken: string) {
// 预检查远程文件URL可用性
await checkFileUrl(fileUrl);

let filename, fileData: Buffer, mimeType;
// 如果是BASE64数据则直接转换为Buffer
if (util.isBASE64Data(fileUrl)) {
mimeType = util.extractBASE64DataFormat(fileUrl);
const ext = mime.getExtension(mimeType);
filename = `${util.uuid()}.${ext}`;
fileData = Buffer.from(util.removeBASE64DataHeader(fileUrl), "base64");
}
// 下载文件到内存，如果您的服务器内存很小，建议考虑改造为流直传到下一个接口上，避免停留占用内存
else {
filename = path.basename(fileUrl);
const queryIndex = filename.indexOf("?");
if (queryIndex != -1) filename = filename.substring(0, queryIndex);
({ data: fileData } = await axios.get(fileUrl, {
responseType: "arraybuffer",
// 100M限制
maxContentLength: FILE_MAX_SIZE,
headers: {
Cookie:
"INGRESSCOOKIE=1711735363.098.26095.391795|cdfd1cd25bff0dd747986e2907e40a4e; Oasis-Webid=9b4315f03ff3e2abdf7a0b6eb1d41b293ac6fa12; Oasis-Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY3RpdmF0ZWQiOmZhbHNlLCJhZ2UiOjgsImJhbmVkIjpmYWxzZSwiZXhwIjoxNzExOTAzODkxLCJtb2RlIjoyLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.KrkngKk8drUVfgRBnEE3A07JKjmqgHL3c7J5PlMxKWw...eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBfaWQiOjEwMzAwLCJkZXZpY2VfaWQiOiI5YjQzMTVmMDNmZjNlMmFiZGY3YTBiNmViMWQ0MWIyOTNhYzZmYTEyIiwiZXhwIjoxNzEzMDMxMzkxLCJvYXNpc19pZCI6ODM1NDA2NzE4ODA5NTM4NTYsInZlcnNpb24iOjF9.HRfkpUOFNGO0Jm6wvijGg9PzxD9d9-j4gXh4eqOkAKk",
Referer: "https://platform.stepfun.com/",
UserAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
},
// 60秒超时
timeout: 60000,
}));
}

// 获取文件的MIME类型
mimeType = mimeType || mime.getType(filename);
// 上传文件到目标OSS
const { deviceId, token } = await acquireToken(refreshToken);
let result = await axios.request({
method: "PUT",
url: `https://stepchat.cn/api/storage?file_name=${filename}`,
data: fileData,
// 100M限制
maxBodyLength: FILE_MAX_SIZE,
// 60秒超时
timeout: 60000,
headers: {
Cookie: generateCookie(deviceId, token),
"Oasis-Webid": deviceId,
Referer: "https://stepchat.cn/chats/new",
"Stepchat-Meta-Width": "undefined",
"Stepchat-Meta-Height": "undefined",
"Stepchat-Meta-Size": fileData.byteLength,
...FAKE_HEADERS,
},
validateStatus: () => true,
});
const { id: fileId } = checkResult(result, refreshToken);

let fileStatus;
const startTime = util.unixTimestamp();
while (fileStatus != 1) {
// 获取文件上传结果
result = await axios.post(
"https://stepchat.cn/api/proto.file.v1.FileService/GetFileStatus",
{
id: fileId,
},
{
headers: {
Cookie: generateCookie(deviceId, token),
"Oasis-Webid": deviceId,
Referer: "https://stepchat.cn/chats/new",
...FAKE_HEADERS,
},
timeout: 15000,
}
);
({ fileStatus } = checkResult(result, refreshToken));
// 上传失败处理
if ([12, 22, 59, 404].includes(fileStatus))
throw new APIException(EX.API_FILE_UPLOAD_FAILED);
// 上传超时处理
if (util.unixTimestamp() - startTime > 60)
throw new APIException(EX.API_FILE_UPLOAD_TIMEOUT);
}

return {
attachmentType: mimeType,
attachmentId: fileId,
name: filename,
width: "undefined",
height: "undefined",
size: `${fileData.byteLength}`,
};
}

/**
* Token切分
*
Expand Down

0 comments on commit 19ecdcf

Please sign in to comment.