Skip to content

feat(backend): concurrent model requests #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 24, 2024
Merged

feat(backend): concurrent model requests #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d5a7b7
refactor(backend): streamline parallel node execution and remove unus…
Sma1lboy Dec 18, 2024
9524e26
refactor(tests): remove unused node types and improve logging with Lo…
Sma1lboy Dec 18, 2024
9b695e4
refactor(types): consolidate GenerateMessageParams and related types …
Sma1lboy Dec 18, 2024
2fbbc57
feat(package): add p-queue dependency for improved task management
Sma1lboy Dec 18, 2024
abe2524
[autofix.ci] apply automated fixes
autofix-ci[bot] Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions backend/src/build-system/__tests__/test-generate-doc.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
{
id: 'op:PRD',
name: 'PRD Generation Node',
type: 'ANALYSIS',
subType: 'PRD',
},
],
},
Expand All @@ -37,8 +35,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
{
id: 'op:UX:SMD',
name: 'UX Sitemap Document Node',
type: 'UX',
subType: 'SITEMAP',
requires: ['op:PRD'],
},
],
Expand All @@ -50,8 +46,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS', () => {
{
id: 'op:UX:SMS',
name: 'UX Sitemap Structure Node',
type: 'UX',
subType: 'VIEWS',
requires: ['op:UX:SMD'],
},
],
Expand Down
11 changes: 0 additions & 11 deletions backend/src/build-system/__tests__/test.backend-code-generator.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:PRD',
name: 'PRD Generation Node',
type: 'ANALYSIS',
subType: 'PRD',
},
],
},
Expand All @@ -39,8 +37,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:UX:SMD',
name: 'UX Sitemap Document Node',
type: 'UX',
subType: 'SITEMAP',
requires: ['op:PRD'],
},
],
Expand All @@ -52,8 +48,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:UX:DATAMAP:DOC',
name: 'UX Data Map Document Node',
type: 'UX',
subType: 'DATAMAP',
requires: ['op:UX:SMD'],
},
],
Expand All @@ -65,8 +59,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:DATABASE_REQ',
name: 'Database Requirements Node',
type: 'DATABASE',
subType: 'SCHEMAS',
requires: ['op:UX:DATAMAP:DOC'],
},
],
Expand All @@ -78,8 +70,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:DATABASE:SCHEMAS',
name: 'Database Schemas Node',
type: 'DATABASE',
subType: 'SCHEMAS',
requires: ['op:DATABASE_REQ'],
},
],
Expand All @@ -91,7 +81,6 @@ describe('Sequence: PRD -> UXSD -> UXDD -> UXSS -> DBSchemas -> BackendCodeGener
{
id: 'op:BACKEND:CODE',
name: 'Backend Code Generator Node',
type: 'BACKEND',
requires: ['op:DATABASE:SCHEMAS', 'op:UX:DATAMAP:DOC'],
},
],
Expand Down
91 changes: 22 additions & 69 deletions backend/src/build-system/__tests__/test.fullstack-gen.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import * as fs from 'fs';
import * as path from 'path';
import { writeToFile } from './utils';
import { BuildMonitor } from '../monitor';
import { Logger } from '@nestjs/common';

describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas -> Frontend_File_struct -> Frontend_File_arch -> BackendCodeGenerator', () => {
// Generate a unique folder with a timestamp
Expand All @@ -24,85 +23,54 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
steps: [
{
id: 'step-1',
name: 'Generate PRD',
name: 'Initial Analysis',
parallel: false,
nodes: [
{
id: 'op:PRD',
name: 'PRD Generation Node',
type: 'ANALYSIS',
subType: 'PRD',
},
],
},
{
id: 'step-2',
name: 'Generate UX Sitemap Document',
name: 'UX Base Document Generation',
parallel: false,
nodes: [
{
id: 'op:UX:SMD',
name: 'UX Sitemap Document Node',
type: 'UX',
subType: 'SITEMAP',
requires: ['op:PRD'],
},
],
},
{
id: 'step-4',
name: 'Generate UX Sitemap Structure',
id: 'step-3',
name: 'Parallel UX Processing',
parallel: true,
nodes: [
{
id: 'op:UX:SMS',
name: 'UX Sitemap Structure Node',
type: 'UX',
subType: 'VIEWS',
requires: ['op:UX:SMD'],
},
],
},
{
id: 'step-5',
name: 'Generate UX Data Map Document',
nodes: [
{
id: 'op:UX:DATAMAP:DOC',
name: 'UX Data Map Document Node',
type: 'UX',
subType: 'DATAMAP',
requires: ['op:UX:SMD'],
},
],
},
{
id: 'step-6',
name: 'Generate Database Requirements',
id: 'step-4',
name: 'Parallel Project Structure',
parallel: true,
nodes: [
{
id: 'op:DATABASE_REQ',
name: 'Database Requirements Node',
type: 'DATABASE',
subType: 'SCHEMAS',
requires: ['op:UX:DATAMAP:DOC'],
},
],
},
{
id: 'step-7',
name: 'Generate Database Schemas',
nodes: [
{
id: 'op:DATABASE:SCHEMAS',
name: 'Database Schemas Node',
type: 'DATABASE',
subType: 'SCHEMAS',
requires: ['op:DATABASE_REQ'],
},
],
},
{
id: 'step-8',
name: 'file structure generation',
nodes: [
{
id: 'op:FILE:STRUCT',
name: 'file structure generation',
Expand All @@ -114,9 +82,15 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
],
},
{
id: 'step-9',
name: 'File_Arch Document',
id: 'step-5',
name: 'Parallel Implementation',
parallel: true,
nodes: [
{
id: 'op:DATABASE:SCHEMAS',
name: 'Database Schemas Node',
requires: ['op:DATABASE_REQ'],
},
{
id: 'op:FILE:ARCH',
name: 'File_Arch',
Expand All @@ -125,29 +99,25 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
],
},
{
id: 'step-10',
name: 'Generate Backend Code',
id: 'step-6',
name: 'Final Code Generation',
parallel: false,
nodes: [
{
id: 'op:BACKEND:CODE',
name: 'Backend Code Generator Node',
type: 'BACKEND',
requires: ['op:DATABASE:SCHEMAS', 'op:UX:DATAMAP:DOC'],
},
],
},
],
};

// Initialize the BuilderContext with the defined sequence and environment
const context = new BuilderContext(sequence, 'test-env');
const monitor = BuildMonitor.getInstance();

try {
console.log('Starting sequence execution...');
console.time('Total Execution Time');

// Execute the build sequence
await context.execute();

console.timeEnd('Total Execution Time');
Expand All @@ -158,8 +128,6 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
monitorReport,
'utf8',
);
console.log('\nExecution Metrics Report:');
console.log(monitorReport);

const sequenceMetrics = monitor.getSequenceMetrics(sequence.id);
if (sequenceMetrics) {
Expand All @@ -184,15 +152,8 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
console.table(metricsJson);
}

console.log('\nProcessing individual node results and metrics...');
for (const step of sequence.steps) {
const stepMetrics = sequenceMetrics?.stepMetrics.get(step.id);
console.log(`\nStep: ${step.name} (${step.id})`);
console.log(`Duration: ${stepMetrics?.duration}ms`);
console.log(
`Completed Nodes: ${stepMetrics?.completedNodes}/${stepMetrics?.totalNodes}`,
);

for (const node of step.nodes) {
const resultData = await context.getNodeData(node.id);
const nodeMetrics = stepMetrics?.nodeMetrics.get(node.id);
Expand Down Expand Up @@ -226,15 +187,7 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
JSON.stringify(summary, null, 2),
'utf8',
);

console.log('\nExecution Summary:');
console.table(summary);

console.log('\nDetailed logs and metrics stored in:', logFolderPath);
} catch (error) {
console.timeEnd('Total Execution Time');

// 保存错误信息和当前的执行状态
const errorReport = {
error: {
message: error.message,
Expand All @@ -258,5 +211,5 @@ describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas ->
);
throw new Error('Sequence execution failed.');
}
}, 600000); // Timeout set to 10 minutes
}, 300000); // Timeout set to 10 minutes
});
5 changes: 3 additions & 2 deletions backend/src/build-system/__tests__/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { Logger } from '@nestjs/common';
import * as fs from 'fs';
import * as path from 'path';
/**
Expand All @@ -20,9 +21,9 @@ export const writeToFile = (

// Write the formatted content to the file
fs.writeFileSync(filePath, formattedContent, 'utf8');
console.log(`Successfully wrote data for ${handlerName} to ${filePath}`);
Logger.log(`Successfully wrote data for ${handlerName} to ${filePath}`);
} catch (error) {
console.error(`Failed to write data for ${handlerName}:`, error);
Logger.error(`Failed to write data for ${handlerName}:`, error);
throw error;
}
};
Expand Down
Loading
Loading