Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions lib/worker/loadTool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { ToolTagEnum } from '@tool/type/tags';
import { existsSync } from 'fs';
import { readdir } from 'fs/promises';
import { join } from 'path';
import { generateToolVersion, generateToolSetVersion } from '@tool/utils/tool';

const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
if (isProd) {
Expand All @@ -28,16 +29,6 @@ const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
const parentIcon = rootMod.icon ?? getIconPath(`${toolsetId}/logo`);

if (isToolSet) {
tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: []
});

const children: ToolType[] = [];

{
Expand All @@ -49,27 +40,54 @@ const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
const toolId = childMod.toolId || `${toolsetId}/${file}`;

const childIcon = childMod.icon ?? rootMod.icon ?? getIconPath(`${toolsetId}/${file}/logo`);

// Generate version for child tool
const childVersion = childMod.versionList
? generateToolVersion(childMod.versionList)
: generateToolVersion([]);

children.push({
...childMod,
toolId,
toolFilename: filename,
icon: childIcon,
parentId: toolsetId
parentId: toolsetId,
version: childVersion
});
}
}

// Generate version for tool set based on children
const toolSetVersion = generateToolSetVersion(children) ?? '';

tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: [],
version: toolSetVersion
});

tools.push(...children);
} else {
// is not toolset
const icon = rootMod.icon ?? getIconPath(`${toolsetId}/logo`);

// Generate version for single tool
const toolVersion = (rootMod as any).versionList
? generateToolVersion((rootMod as any).versionList)
: generateToolVersion([]);

tools.push({
...(rootMod as ToolType),
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon,
toolFilename: filename
toolFilename: filename,
version: toolVersion
});
}

Expand Down
42 changes: 30 additions & 12 deletions modules/tool/build/build-json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { existsSync, writeFileSync } from 'fs';
import { readdir } from 'fs/promises';
import { join } from 'path';
import { ToolDetailSchema } from 'sdk/client';
import { generateToolVersion, generateToolSetVersion } from '../utils/tool';

const filterToolList = ['.DS_Store', '.git', '.github', 'node_modules', 'dist', 'scripts'];

Expand All @@ -26,16 +27,6 @@ const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
const parentIcon = rootMod.icon ?? `${S3BasePath}${UploadToolsS3Path}/${toolsetId}/logo`;

if (isToolSet) {
tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: []
});

const children: ToolType[] = [];

{
Expand All @@ -50,27 +41,54 @@ const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
childMod.icon ??
rootMod.icon ??
`${S3BasePath}${UploadToolsS3Path}/${toolsetId}/${file}/logo`;

// Generate version for child tool
const childVersion = childMod.versionList
? generateToolVersion(childMod.versionList)
: generateToolVersion([]);

children.push({
...childMod,
toolId,
toolFilename: filename,
icon: childIcon,
parentId: toolsetId
parentId: toolsetId,
version: childVersion
});
}
}

// Generate version for tool set based on children
const toolSetVersion = generateToolSetVersion(children) ?? '';

tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: [],
version: toolSetVersion
});

tools.push(...children);
} else {
// is not toolset
const icon = rootMod.icon ?? `${S3BasePath}${UploadToolsS3Path}/${toolsetId}/logo`;

// Generate version for single tool
const toolVersion = (rootMod as any).versionList
? generateToolVersion((rootMod as any).versionList)
: generateToolVersion([]);

tools.push({
...(rootMod as ToolType),
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon,
toolFilename: filename
toolFilename: filename,
version: toolVersion
});
}

Expand Down
4 changes: 2 additions & 2 deletions modules/tool/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,11 @@ export async function initTools() {
}

addLog.info(`Load Tools: ${toolMap.size}`);
isIniting = false;
global.isIniting = false;
return toolMap;
} catch (e) {
addLog.error(`Init Tools Error:`, e);
isIniting = false;
global.isIniting = false;
return getCachedData(SystemCacheKeyEnum.systemTool);
}
}
42 changes: 30 additions & 12 deletions modules/tool/loadToolDev.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { ToolType, ToolSetType } from './type';
import { ToolTagEnum } from './type/tags';
import { publicS3Server } from '@/s3';
import { mimeMap } from '@/s3/const';
import { generateToolVersion, generateToolSetVersion } from './utils/tool';

/**
* Load Tools in dev mode. Only avaliable in dev mode
Expand Down Expand Up @@ -82,16 +83,6 @@ export const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
(await publicS3Server.generateExternalUrl(`${UploadToolsS3Path}/${toolsetId}/logo`));

if (isToolSet) {
tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: []
});

const children: ToolType[] = [];

{
Expand Down Expand Up @@ -161,29 +152,56 @@ export const LoadToolsDev = async (filename: string): Promise<ToolType[]> => {
(await publicS3Server.generateExternalUrl(
`${UploadToolsS3Path}/${toolsetId}/${file}/logo`
));

// Generate version for child tool
const childVersion = childMod.versionList
? generateToolVersion(childMod.versionList)
: generateToolVersion([]);

children.push({
...childMod,
toolId,
toolFilename: filename,
icon: childIcon,
parentId: toolsetId
parentId: toolsetId,
version: childVersion
});
}
}

// Generate version for tool set based on children
const toolSetVersion = generateToolSetVersion(children);

tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: filename,
cb: () => Promise.resolve({}),
versionList: [],
version: toolSetVersion ?? ''
});

tools.push(...children);
} else {
// is not toolset
const icon =
rootMod.icon ??
(await publicS3Server.generateExternalUrl(`${UploadToolsS3Path}/${toolsetId}/logo`));

// Generate version for single tool
const toolVersion = (rootMod as any).versionList
? generateToolVersion((rootMod as any).versionList)
: generateToolVersion([]);

tools.push({
...(rootMod as ToolType),
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon,
toolFilename: filename
toolFilename: filename,
version: toolVersion
});
}

Expand Down
32 changes: 19 additions & 13 deletions modules/tool/parseMod.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ToolTagEnum } from 'sdk/client';
import { UploadToolsS3Path } from './constants';
import type { ToolSetType, ToolType } from './type';
import { PublicBucketBaseURL } from '@/s3/const';
import { generateToolVersion, generateToolSetVersion } from './utils/tool';

export const getIconPath = (name: string) => `${PublicBucketBaseURL}${UploadToolsS3Path}/${name}`;

Expand All @@ -21,24 +22,15 @@ export const parseMod = async ({

const parentIcon = rootMod.icon || getIconPath(`${toolsetId}/logo`);

// push parent
tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: `${filename}`,
cb: () => Promise.resolve({}),
versionList: []
});

const children = rootMod.children;

for (const child of children) {
const childToolId = child.toolId;

const childIcon = child.icon || rootMod.icon || getIconPath(`${childToolId}/logo`);

// Generate version for child tool
const childVersion = generateToolVersion(child.versionList);
tools.push({
...child,
toolId: childToolId,
Expand All @@ -47,9 +39,22 @@ export const parseMod = async ({
courseUrl: rootMod.courseUrl,
author: rootMod.author,
icon: childIcon,
toolFilename: filename
toolFilename: filename,
version: childVersion
});
}

// push parent
tools.push({
...rootMod,
tags: rootMod.tags || [ToolTagEnum.enum.other],
toolId: toolsetId,
icon: parentIcon,
toolFilename: `${filename}`,
cb: () => Promise.resolve({}),
versionList: [],
version: generateToolSetVersion(children) || ''
});
} else {
// is not toolset
const toolId = rootMod.toolId;
Expand All @@ -61,7 +66,8 @@ export const parseMod = async ({
tags: rootMod.tags || [ToolTagEnum.enum.tools],
icon,
toolId,
toolFilename: filename
toolFilename: filename,
version: generateToolVersion(rootMod.versionList)
});
}
return tools;
Expand Down
1 change: 1 addition & 0 deletions modules/tool/type/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export const ToolSchema = ToolConfigWithCbSchema.extend({
parentId: z.string().optional().describe('The parent id of the tool'),
toolFilename: z.string(),

version: z.string().describe('The version hash of the tool'),
// ToolSet Parent
secretInputConfig: z
.array(InputConfigSchema)
Expand Down
42 changes: 24 additions & 18 deletions modules/tool/utils/tool.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import type { z } from 'zod';
import type { ToolSetConfigType } from '@tool/type';
import type { ToolSetConfigType, ToolType, ToolSetType } from '@tool/type';
import { ToolConfigSchema } from '@tool/type/tool';
import type { RunToolSecondParamsType } from '@tool/type/req';
import { createHash } from 'node:crypto';

export const exportTool = <T extends z.Schema, D extends z.Schema>({
toolCb,
Expand Down Expand Up @@ -46,20 +47,25 @@ export const exportToolSet = ({ config }: { config: ToolSetConfigType }) => {
};
};

// export function formatToolList(
// list: (z.infer<typeof ToolSchema> | z.infer<typeof ToolSetSchema>)[]
// ): ToolListItemType[] {
// return list.map((item) => ({
// author: item.author,
// name: item.name,
// parentId: 'parentId' in item ? item.parentId : undefined,
// courseUrl: item.courseUrl,
// id: item.toolId,
// avatar: item.icon,
// versionList: item.versionList,
// description: item.description,
// toolDescription: item.toolDescription,
// templateType: item.tags?.[0],
// secretInputConfig: item.secretInputConfig
// }));
// }
export function generateToolVersion(versionList: Array<{ value: string }>): string {
const versionString = versionList.map((v) => v.value).join('');
return createHash('sha256').update(versionString).digest('hex').substring(0, 8);
}

/**
* Generate version hash for a tool set based on all child tools' versions
* @param children - Array of child tools
* @returns First 8 characters of SHA256 hash of all child version hashes concatenated
*/
export function generateToolSetVersion(children: ToolType[]) {
if (!children || children.length === 0) {
return undefined;
}

const childVersions = children
.map((child) => generateToolVersion(child.versionList) || '')
.sort();
const versionString = childVersions.join('');

return createHash('sha256').update(versionString).digest('hex').substring(0, 8);
}
Loading