From d5e347940c3ff0224a40a93c42c131362a9f62d0 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:23:40 +0200 Subject: [PATCH 01/12] docs: add design spec and implementation plan for run_plan_pipeline split Design spec and 11-task implementation plan for splitting the 4,257-line run_plan_pipeline.py into ~66 individual stage files under stages/. Optimized for parallel agent work, self_improve/ integration, and conflict-free DAG insertion. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-04-02-split-run-plan-pipeline.md | 1075 +++++++++++++++++ ...26-04-02-split-run-plan-pipeline-design.md | 233 ++++ 2 files changed, 1308 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-02-split-run-plan-pipeline.md create mode 100644 docs/superpowers/specs/2026-04-02-split-run-plan-pipeline-design.md diff --git a/docs/superpowers/plans/2026-04-02-split-run-plan-pipeline.md b/docs/superpowers/plans/2026-04-02-split-run-plan-pipeline.md new file mode 100644 index 000000000..3545959ab --- /dev/null +++ b/docs/superpowers/plans/2026-04-02-split-run-plan-pipeline.md @@ -0,0 +1,1075 @@ +# Split run_plan_pipeline.py Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extract ~66 Luigi task classes from `run_plan_pipeline.py` (4,257 lines) into individual files under `stages/`, leaving only the shared framework in the original file. + +**Architecture:** Each task class moves verbatim to its own file under `worker_plan/worker_plan_internal/plan/stages/`. Each file imports `PlanTask` from `run_plan_pipeline` and its upstream task dependencies from sibling stage files. `FullPlanPipeline` moves to `stages/full_plan_pipeline.py`. `run_plan_pipeline.py` shrinks to ~280 lines keeping only `PlanTask`, `ExecutePipeline`, and supporting utilities. + +**Tech Stack:** Python 3.11, Luigi (task orchestration), llama_index + +**Spec:** `docs/superpowers/specs/2026-04-02-split-run-plan-pipeline-design.md` + +--- + +## Reference: Source File + +All task classes are extracted verbatim from: +`worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` + +The source file is referred to as `SOURCE` below. When a step says "copy lines X-Y from SOURCE", it means copy those lines verbatim — no modifications to the task class body. + +## How Each Stage File Is Structured + +Every stage file follows this exact pattern: + +```python +"""Pipeline stage: .""" + + +logger = logging.getLogger(__name__) # only if the task class uses logger + +class FooTask(PlanTask): + # ... verbatim from SOURCE +``` + +The import block is the ONLY new code per file. The class body is a verbatim copy. + +--- + +### Task 1: Create stages/ directory and Phase 1-2 stages (Input, Validation, Purpose) + +**Files:** +- Create: `worker_plan/worker_plan_internal/plan/stages/__init__.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/start_time.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/setup.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/redline_gate.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/premise_attack.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/identify_purpose.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/plan_type.py` + +- [ ] **Step 1: Create the stages directory with empty `__init__.py`** + +```python +# worker_plan/worker_plan_internal/plan/stages/__init__.py +``` + +(Empty file — stage files are imported directly by whoever needs them.) + +- [ ] **Step 2: Create `stages/start_time.py`** + +```python +"""Pipeline stage: record pipeline start timestamp.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum +``` + +Copy the `StartTimeTask` class from SOURCE lines 217-225 verbatim after the imports. + +- [ ] **Step 3: Create `stages/setup.py`** + +```python +"""Pipeline stage: load initial plan prompt.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum +``` + +Copy `SetupTask` from SOURCE lines 228-236 verbatim. + +- [ ] **Step 4: Create `stages/redline_gate.py`** + +```python +"""Pipeline stage: preliminary validation of the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.diagnostics.redline_gate import RedlineGate +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +``` + +Copy `RedlineGateTask` from SOURCE lines 239-260 verbatim. + +- [ ] **Step 5: Create `stages/premise_attack.py`** + +```python +"""Pipeline stage: challenge foundational assumptions.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.diagnostics.premise_attack import PremiseAttack +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +``` + +Copy `PremiseAttackTask` from SOURCE lines 263-286 verbatim. + +- [ ] **Step 6: Create `stages/identify_purpose.py`** + +```python +"""Pipeline stage: determine plan purpose (business/personal/other).""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_purpose import IdentifyPurpose +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +``` + +Copy `IdentifyPurposeTask` from SOURCE lines 289-313 verbatim. + +- [ ] **Step 7: Create `stages/plan_type.py`** + +```python +"""Pipeline stage: determine if plan is digital-only or physical.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_plan_type import IdentifyPlanType +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +``` + +Copy `PlanTypeTask` from SOURCE lines 316-350 verbatim. + +- [ ] **Step 8: Verify imports resolve** + +Run from `worker_plan/`: +```bash +python -c "from worker_plan_internal.plan.stages.plan_type import PlanTypeTask; print('OK')" +``` +Expected: `OK` + +- [ ] **Step 9: Commit** + +```bash +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 1-2 pipeline stages to individual files + +Extract StartTimeTask, SetupTask, RedlineGateTask, PremiseAttackTask, +IdentifyPurposeTask, PlanTypeTask into stages/ directory." +``` + +--- + +### Task 2: Phase 3 stages (Strategic Options & Scenarios) + +**Files:** +- Create: `worker_plan/worker_plan_internal/plan/stages/potential_levers.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/deduplicate_levers.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/enrich_levers.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/focus_on_vital_few_levers.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/strategic_decisions_markdown.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/candidate_scenarios.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/select_scenario.py` +- Create: `worker_plan/worker_plan_internal/plan/stages/scenarios_markdown.py` + +- [ ] **Step 1: Create `stages/potential_levers.py`** + +```python +"""Pipeline stage: identify potential strategic levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.identify_potential_levers import IdentifyPotentialLevers +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +``` + +Copy `PotentialLeversTask` from SOURCE lines 352-392 verbatim. + +- [ ] **Step 2: Create `stages/deduplicate_levers.py`** + +```python +"""Pipeline stage: remove redundant levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.deduplicate_levers import DeduplicateLevers +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.potential_levers import PotentialLeversTask +``` + +Copy `DeduplicateLeversTask` from SOURCE lines 395-439 verbatim. + +- [ ] **Step 3: Create `stages/enrich_levers.py`** + +```python +"""Pipeline stage: enrich levers with additional information.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.enrich_potential_levers import EnrichPotentialLevers +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.deduplicate_levers import DeduplicateLeversTask +``` + +Copy `EnrichLeversTask` from SOURCE lines 441-486 verbatim. + +- [ ] **Step 4: Create `stages/focus_on_vital_few_levers.py`** + +```python +"""Pipeline stage: apply 80/20 principle to select vital levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.focus_on_vital_few_levers import FocusOnVitalFewLevers +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask +``` + +Copy `FocusOnVitalFewLeversTask` from SOURCE lines 488-532 verbatim. + +- [ ] **Step 5: Create `stages/strategic_decisions_markdown.py`** + +```python +"""Pipeline stage: consolidate strategic decisions to markdown.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.strategic_decisions_markdown import StrategicDecisionsMarkdown +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +``` + +Copy `StrategicDecisionsMarkdownTask` from SOURCE lines 535-561 verbatim. + +- [ ] **Step 6: Create `stages/candidate_scenarios.py`** + +```python +"""Pipeline stage: generate candidate scenarios from vital levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.candidate_scenarios import CandidateScenarios +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +``` + +Copy `CandidateScenariosTask` from SOURCE lines 563-610 verbatim. + +- [ ] **Step 7: Create `stages/select_scenario.py`** + +```python +"""Pipeline stage: select the best scenario.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.select_scenario import SelectScenario +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask +``` + +Copy `SelectScenarioTask` from SOURCE lines 613-665 verbatim. + +- [ ] **Step 8: Create `stages/scenarios_markdown.py`** + +```python +"""Pipeline stage: present scenarios in human-readable markdown.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.scenarios_markdown import ScenariosMarkdown +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask +from worker_plan_internal.plan.stages.select_scenario import SelectScenarioTask +``` + +Copy `ScenariosMarkdownTask` from SOURCE lines 668-695 verbatim. + +- [ ] **Step 9: Verify imports resolve** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask; print('OK')" +``` + +- [ ] **Step 10: Commit** + +```bash +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 3 pipeline stages (levers & scenarios)" +``` + +--- + +### Task 3: Phase 4-5 stages (Context & Assumptions) + +**Files:** +- Create: `stages/physical_locations.py` — `PhysicalLocationsTask` (SOURCE lines 698-761) +- Create: `stages/currency_strategy.py` — `CurrencyStrategyTask` (SOURCE lines 763-813) +- Create: `stages/identify_risks.py` — `IdentifyRisksTask` (SOURCE lines 816-870) +- Create: `stages/make_assumptions.py` — `MakeAssumptionsTask` (SOURCE lines 873-934) +- Create: `stages/distill_assumptions.py` — `DistillAssumptionsTask` (SOURCE lines 937-984) +- Create: `stages/review_assumptions.py` — `ReviewAssumptionsTask` (SOURCE lines 987-1047) +- Create: `stages/consolidate_assumptions_markdown.py` — `ConsolidateAssumptionsMarkdownTask` (SOURCE lines 1050-1132) + +Each file needs these common imports plus task-specific ones: + +- [ ] **Step 1: Create `stages/physical_locations.py`** + +```python +"""Pipeline stage: identify physical locations for the plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.physical_locations import PhysicalLocations +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask + +logger = logging.getLogger(__name__) +``` + +Copy `PhysicalLocationsTask` from SOURCE lines 698-761 verbatim. + +- [ ] **Step 2: Create `stages/currency_strategy.py`** + +```python +"""Pipeline stage: determine currency and financial strategy.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.currency_strategy import CurrencyStrategy +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +``` + +Copy `CurrencyStrategyTask` from SOURCE lines 763-813 verbatim. + +- [ ] **Step 3: Create `stages/identify_risks.py`** + +```python +"""Pipeline stage: identify risks based on locations and strategy.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_risks import IdentifyRisks +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +``` + +Copy `IdentifyRisksTask` from SOURCE lines 816-870 verbatim. + +- [ ] **Step 4: Create `stages/make_assumptions.py`** + +```python +"""Pipeline stage: make initial assumptions about the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.make_assumptions import MakeAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +``` + +Copy `MakeAssumptionsTask` from SOURCE lines 873-934 verbatim. + +- [ ] **Step 5: Create `stages/distill_assumptions.py`** + +```python +"""Pipeline stage: distill and consolidate raw assumptions.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.distill_assumptions import DistillAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +``` + +Copy `DistillAssumptionsTask` from SOURCE lines 937-984 verbatim. + +- [ ] **Step 6: Create `stages/review_assumptions.py`** + +```python +"""Pipeline stage: review and find issues with assumptions.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.review_assumptions import ReviewAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask + +logger = logging.getLogger(__name__) +``` + +Copy `ReviewAssumptionsTask` from SOURCE lines 987-1047 verbatim. + +- [ ] **Step 7: Create `stages/consolidate_assumptions_markdown.py`** + +```python +"""Pipeline stage: combine assumption documents into one markdown.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.shorten_markdown import ShortenMarkdown +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask +from worker_plan_internal.plan.stages.review_assumptions import ReviewAssumptionsTask + +logger = logging.getLogger(__name__) +``` + +Copy `ConsolidateAssumptionsMarkdownTask` from SOURCE lines 1050-1132 verbatim. + +- [ ] **Step 8: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 4-5 pipeline stages (context & assumptions)" +``` + +--- + +### Task 4: Phase 6-7 stages (Plan Foundation & Governance) + +**Files:** +- Create: `stages/pre_project_assessment.py` — `PreProjectAssessmentTask` (SOURCE lines 1135-1183) +- Create: `stages/project_plan.py` — `ProjectPlanTask` (SOURCE lines 1185-1241) +- Create: `stages/governance_phase1_audit.py` — `GovernancePhase1AuditTask` (SOURCE lines 1244-1291) +- Create: `stages/governance_phase2_bodies.py` — `GovernancePhase2BodiesTask` (SOURCE lines 1294-1345) +- Create: `stages/governance_phase3_impl_plan.py` — `GovernancePhase3ImplPlanTask` (SOURCE lines 1348-1399) +- Create: `stages/governance_phase4_decision_escalation_matrix.py` — `GovernancePhase4DecisionEscalationMatrixTask` (SOURCE lines 1401-1456) +- Create: `stages/governance_phase5_monitoring_progress.py` — `GovernancePhase5MonitoringProgressTask` (SOURCE lines 1458-1517) +- Create: `stages/governance_phase6_extra.py` — `GovernancePhase6ExtraTask` (SOURCE lines 1519-1586) +- Create: `stages/consolidate_governance.py` — `ConsolidateGovernanceTask` (SOURCE lines 1588-1629) + +For each file, follow this pattern: + +- [ ] **Step 1: Create `stages/pre_project_assessment.py`** + +```python +"""Pipeline stage: pre-project feasibility assessment.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.expert.pre_project_assessment import PreProjectAssessment +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask + +logger = logging.getLogger(__name__) +``` + +Copy `PreProjectAssessmentTask` from SOURCE lines 1135-1183 verbatim. + +- [ ] **Step 2: Create `stages/project_plan.py`** + +```python +"""Pipeline stage: generate the main project plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.project_plan import ProjectPlan +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask + +logger = logging.getLogger(__name__) +``` + +Copy `ProjectPlanTask` from SOURCE lines 1185-1241 verbatim. + +- [ ] **Step 3: Create all 7 governance stage files** + +Each governance file follows the same pattern. The imports for each are: + +**`stages/governance_phase1_audit.py`** (SOURCE lines 1244-1291): +```python +"""Pipeline stage: governance audit.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase1_audit import GovernancePhase1Audit +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask + +logger = logging.getLogger(__name__) +``` + +**`stages/governance_phase2_bodies.py`** (SOURCE lines 1294-1345): +```python +"""Pipeline stage: identify governance bodies.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase2_bodies import GovernancePhase2Bodies +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask + +logger = logging.getLogger(__name__) +``` + +**`stages/governance_phase3_impl_plan.py`** (SOURCE lines 1348-1399): +```python +"""Pipeline stage: governance implementation plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase3_impl_plan import GovernancePhase3ImplPlan +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask + +logger = logging.getLogger(__name__) +``` + +**`stages/governance_phase4_decision_escalation_matrix.py`** (SOURCE lines 1401-1456): +Same pattern. Imports `GovernancePhase4DecisionEscalationMatrix` from `worker_plan_internal.governance.governance_phase4_decision_escalation_matrix`. Upstream tasks: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, GovernancePhase2BodiesTask, GovernancePhase3ImplPlanTask. Uses `json`, `format_json_for_use_in_query`. + +**`stages/governance_phase5_monitoring_progress.py`** (SOURCE lines 1458-1517): +Same pattern. Imports `GovernancePhase5MonitoringProgress`. Upstream adds `GovernancePhase4DecisionEscalationMatrixTask`. Uses `json`, `format_json_for_use_in_query`. + +**`stages/governance_phase6_extra.py`** (SOURCE lines 1519-1586): +Same pattern. Imports `GovernancePhase6Extra`. Upstream includes all governance phases 1-5. Uses `json`, `format_json_for_use_in_query`. + +**`stages/consolidate_governance.py`** (SOURCE lines 1588-1629): +```python +"""Pipeline stage: consolidate all governance phases into one document.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask +from worker_plan_internal.plan.stages.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgressTask +from worker_plan_internal.plan.stages.governance_phase6_extra import GovernancePhase6ExtraTask +``` + +- [ ] **Step 4: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 6-7 pipeline stages (plan foundation & governance)" +``` + +--- + +### Task 5: Phase 8 stages (Team) + +**Files:** +- Create: `stages/related_resources.py` — `RelatedResourcesTask` (SOURCE lines 1631-1678) +- Create: `stages/find_team_members.py` — `FindTeamMembersTask` (SOURCE lines 1680-1741) +- Create: `stages/enrich_team_contract_type.py` — `EnrichTeamMembersWithContractTypeTask` (SOURCE lines 1743-1808) +- Create: `stages/enrich_team_background_story.py` — `EnrichTeamMembersWithBackgroundStoryTask` (SOURCE lines 1810-1887) +- Create: `stages/enrich_team_environment_info.py` — `EnrichTeamMembersWithEnvironmentInfoTask` (SOURCE lines 1889-1954) +- Create: `stages/review_team.py` — `ReviewTeamTask` (SOURCE lines 1956-2020) +- Create: `stages/team_markdown.py` — `TeamMarkdownTask` (SOURCE lines 2022-2053) + +Each file imports `PlanTask`, its executor class, `FilenameEnum`, and its upstream task(s) from sibling stages. + +Key imports per file: + +- `related_resources.py`: executor `RelatedResources` from `worker_plan_internal.plan.related_resources`. Upstreams: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `find_team_members.py`: executor `FindTeamMembers` from `worker_plan_internal.team.find_team_members`. Upstreams: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, PreProjectAssessmentTask, ProjectPlanTask, RelatedResourcesTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `enrich_team_contract_type.py`: executor `EnrichTeamMembersWithContractType`. Upstreams: same as find + FindTeamMembersTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `enrich_team_background_story.py`: executor `EnrichTeamMembersWithBackgroundStory`. Upstreams: same + EnrichTeamMembersWithContractTypeTask. Also imports `SpeedVsDetailEnum` from `worker_plan_api.speedvsdetail` (for FAST_BUT_SKIP_DETAILS check). Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `enrich_team_environment_info.py`: executor `EnrichTeamMembersWithEnvironmentInfo`. Upstreams: same but depends on BackgroundStoryTask instead of ContractTypeTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `review_team.py`: executor `ReviewTeam` + `TeamMarkdownDocumentBuilder` from `worker_plan_internal.team.team_markdown_document`. Upstreams: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, PreProjectAssessmentTask, ProjectPlanTask, EnrichTeamMembersWithEnvironmentInfoTask, RelatedResourcesTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `team_markdown.py`: `TeamMarkdownDocumentBuilder` from `worker_plan_internal.team.team_markdown_document`. Upstreams: EnrichTeamMembersWithEnvironmentInfoTask, ReviewTeamTask. Uses `json`, `logging`. + +- [ ] **Step 1: Create all 7 team stage files following the pattern above** + +- [ ] **Step 2: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 8 pipeline stages (team)" +``` + +--- + +### Task 6: Phase 9-10 stages (Analysis & Documents) + +**Files:** +- Create: `stages/swot_analysis.py` — `SWOTAnalysisTask` (SOURCE lines 2055-2123) +- Create: `stages/expert_review.py` — `ExpertReviewTask` (SOURCE lines 2125-2194) +- Create: `stages/data_collection.py` — `DataCollectionTask` (SOURCE lines 2197-2257) +- Create: `stages/identify_documents.py` — `IdentifyDocumentsTask` (SOURCE lines 2259-2328) +- Create: `stages/filter_documents_to_find.py` — `FilterDocumentsToFindTask` (SOURCE lines 2330-2387) +- Create: `stages/filter_documents_to_create.py` — `FilterDocumentsToCreateTask` (SOURCE lines 2389-2446) +- Create: `stages/draft_documents_to_find.py` — `DraftDocumentsToFindTask` (SOURCE lines 2448-2532) +- Create: `stages/draft_documents_to_create.py` — `DraftDocumentsToCreateTask` (SOURCE lines 2534-2618) +- Create: `stages/markdown_documents.py` — `MarkdownWithDocumentsToCreateAndFindTask` (SOURCE lines 2620-2656) + +Key imports per file: + +- `swot_analysis.py`: executor `SWOTAnalysis` from `worker_plan_internal.swot.swot_analysis`. Upstreams: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, IdentifyPurposeTask, ConsolidateAssumptionsMarkdownTask, PreProjectAssessmentTask, ProjectPlanTask, RelatedResourcesTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `expert_review.py`: executors `ExpertFinder` from `worker_plan_internal.expert.expert_finder`, `ExpertCriticism` from `worker_plan_internal.expert.expert_criticism`, `ExpertOrchestrator` from `worker_plan_internal.expert.expert_orchestrator`. Also `LLMExecutor` from `worker_plan_internal.llm_util.llm_executor`. Upstreams: SetupTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, PreProjectAssessmentTask, ProjectPlanTask, SWOTAnalysisTask. Uses `json`, `logging`, `format_json_for_use_in_query`, `FilenameEnum`. + +- `data_collection.py`: executor `DataCollection` from `worker_plan_internal.plan.data_collection`. Upstreams: StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, RelatedResourcesTask, SWOTAnalysisTask, TeamMarkdownTask, ExpertReviewTask. + +- `identify_documents.py`: executor `IdentifyDocuments` from `worker_plan_internal.document.identify_documents`. Upstreams: IdentifyPurposeTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, RelatedResourcesTask, SWOTAnalysisTask, TeamMarkdownTask, ExpertReviewTask. Uses `json`. + +- `filter_documents_to_find.py`: executor `FilterDocumentsToFind` from `worker_plan_internal.document.filter_documents_to_find`. Upstreams: IdentifyPurposeTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, IdentifyDocumentsTask. Uses `json`. + +- `filter_documents_to_create.py`: executor `FilterDocumentsToCreate` from `worker_plan_internal.document.filter_documents_to_create`. Same upstreams as filter_to_find. + +- `draft_documents_to_find.py`: executor `DraftDocumentToFind` from `worker_plan_internal.document.draft_document_to_find`. Also `LLMExecutor, PipelineStopRequested` from `worker_plan_internal.llm_util.llm_executor`, `SpeedVsDetailEnum`. Upstreams: IdentifyPurposeTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, FilterDocumentsToFindTask. Uses `json`, `logging`. + +- `draft_documents_to_create.py`: executor `DraftDocumentToCreate` from `worker_plan_internal.document.draft_document_to_create`. Same pattern as draft_to_find, but depends on FilterDocumentsToCreateTask. + +- `markdown_documents.py`: `markdown_rows_with_document_to_create, markdown_rows_with_document_to_find` from `worker_plan_internal.document.markdown_with_document`. Upstreams: DraftDocumentsToCreateTask, DraftDocumentsToFindTask. Uses `json`. + +- [ ] **Step 1: Create all 9 files** + +- [ ] **Step 2: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 9-10 pipeline stages (analysis & documents)" +``` + +--- + +### Task 7: Phase 11 stages (WBS & Pitch) + +**Files:** +- Create: `stages/create_wbs_level1.py` — `CreateWBSLevel1Task` (SOURCE lines 2658-2706) +- Create: `stages/create_wbs_level2.py` — `CreateWBSLevel2Task` (SOURCE lines 2708-2768) +- Create: `stages/wbs_project_level1_and_level2.py` — `WBSProjectLevel1AndLevel2Task` (SOURCE lines 2770-2795) +- Create: `stages/create_pitch.py` — `CreatePitchTask` (SOURCE lines 2797-2853) +- Create: `stages/convert_pitch_to_markdown.py` — `ConvertPitchToMarkdownTask` (SOURCE lines 2855-2888) +- Create: `stages/identify_task_dependencies.py` — `IdentifyTaskDependenciesTask` (SOURCE lines 2891-2939) +- Create: `stages/estimate_task_durations.py` — `EstimateTaskDurationsTask` (SOURCE lines 2941-3041) +- Create: `stages/create_wbs_level3.py` — `CreateWBSLevel3Task` (SOURCE lines 3043-3158) +- Create: `stages/wbs_project_level1_level2_level3.py` — `WBSProjectLevel1AndLevel2AndLevel3Task` (SOURCE lines 3160-3195) + +Key imports: + +- `create_wbs_level1.py`: executor `CreateWBSLevel1` from `worker_plan_internal.plan.create_wbs_level1`. Upstreams: ProjectPlanTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `create_wbs_level2.py`: executor `CreateWBSLevel2`. Upstreams: StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ProjectPlanTask, CreateWBSLevel1Task, DataCollectionTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `wbs_project_level1_and_level2.py`: `WBSPopulate` from `worker_plan_internal.wbs.wbs_populate`. Upstreams: CreateWBSLevel1Task, CreateWBSLevel2Task. Uses `json`. + +- `create_pitch.py`: executor `CreatePitch` from `worker_plan_internal.pitch.create_pitch`, `WBSProject` from `worker_plan_internal.wbs.wbs_task`. Upstreams: StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ProjectPlanTask, WBSProjectLevel1AndLevel2Task, RelatedResourcesTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `convert_pitch_to_markdown.py`: executor `ConvertPitchToMarkdown` from `worker_plan_internal.pitch.convert_pitch_to_markdown`. Upstreams: CreatePitchTask. Uses `json`, `format_json_for_use_in_query`. + +- `identify_task_dependencies.py`: executor `IdentifyWBSTaskDependencies` from `worker_plan_internal.plan.identify_wbs_task_dependencies`. Upstreams: StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ProjectPlanTask, CreateWBSLevel2Task, DataCollectionTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `estimate_task_durations.py`: executor `EstimateWBSTaskDurations` from `worker_plan_internal.plan.estimate_wbs_task_durations`, `WBSProject` from `worker_plan_internal.wbs.wbs_task`. Also `LLMExecutor, PipelineStopRequested` from `worker_plan_internal.llm_util.llm_executor`, `SpeedVsDetailEnum`. Upstreams: ProjectPlanTask, WBSProjectLevel1AndLevel2Task. Uses `json`, `logging`. + +- `create_wbs_level3.py`: executor `CreateWBSLevel3` from `worker_plan_internal.plan.create_wbs_level3`, `WBSProject` from `worker_plan_internal.wbs.wbs_task`, `WBSPopulate` from `worker_plan_internal.wbs.wbs_populate`. Also `LLMExecutor, PipelineStopRequested`, `SpeedVsDetailEnum`. Upstreams: ProjectPlanTask, WBSProjectLevel1AndLevel2Task, EstimateTaskDurationsTask, DataCollectionTask. Uses `json`, `logging`, `format_json_for_use_in_query`. + +- `wbs_project_level1_level2_level3.py`: `WBSProject` from `worker_plan_internal.wbs.wbs_task`, `WBSPopulate` from `worker_plan_internal.wbs.wbs_populate`. Upstreams: WBSProjectLevel1AndLevel2Task, CreateWBSLevel3Task. Uses `json`. + +- [ ] **Step 1: Create all 9 files** + +- [ ] **Step 2: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 11 pipeline stages (WBS & pitch)" +``` + +--- + +### Task 8: Phase 12-13 stages (Schedule, Final Review & Report) + +**Files:** +- Create: `stages/create_schedule.py` — `CreateScheduleTask` (SOURCE lines 3197-3288) +- Create: `stages/review_plan.py` — `ReviewPlanTask` (SOURCE lines 3290-3364) +- Create: `stages/executive_summary.py` — `ExecutiveSummaryTask` (SOURCE lines 3367-3443) +- Create: `stages/questions_and_answers.py` — `QuestionsAndAnswersTask` (SOURCE lines 3446-3524) +- Create: `stages/premortem.py` — `PremortemTask` (SOURCE lines 3526-3607) +- Create: `stages/self_audit.py` — `SelfAuditTask` (SOURCE lines 3610-3707) +- Create: `stages/report.py` — `ReportTask` (SOURCE lines 3709-3774) + +Key imports: + +- `create_schedule.py`: `ProjectSchedulePopulator, ProjectSchedule` from schedule modules, `ExportGanttDHTMLX, ExportGanttCSV` from schedule modules, `WBSProject` from `worker_plan_internal.wbs.wbs_task`, `WBSTaskTooltip` from `worker_plan_internal.wbs.wbs_task_tooltip`, `PIPELINE_CONFIG` from `worker_plan_internal.plan.pipeline_config`. Upstreams: StartTimeTask, CreateWBSLevel1Task, IdentifyTaskDependenciesTask, EstimateTaskDurationsTask, WBSProjectLevel1AndLevel2AndLevel3Task. Uses `json`, `logging`, `datetime`, `typing.Any`. + +- `review_plan.py`: executor `ReviewPlan` from `worker_plan_internal.plan.review_plan`. Also `LLMExecutor`. Upstreams: StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, ProjectPlanTask, DataCollectionTask, RelatedResourcesTask, SWOTAnalysisTask, TeamMarkdownTask, ConvertPitchToMarkdownTask, ExpertReviewTask, WBSProjectLevel1AndLevel2AndLevel3Task. + +- `executive_summary.py`: executor `ExecutiveSummary` from `worker_plan_internal.plan.executive_summary`. Same upstreams as review_plan + ReviewPlanTask. + +- `questions_and_answers.py`: executor `QuestionsAnswers` from `worker_plan_internal.questions_answers.questions_answers`. Upstreams include ConsolidateGovernanceTask, MarkdownWithDocumentsToCreateAndFindTask, ReviewPlanTask. + +- `premortem.py`: executor `Premortem` from `worker_plan_internal.diagnostics.premortem`. Also `LLMExecutor` and `SpeedVsDetailEnum`. Same broad upstreams + QuestionsAndAnswersTask. + +- `self_audit.py`: executor `SelfAudit` from `worker_plan_internal.self_audit.self_audit`. Also `LLMExecutor`, `SpeedVsDetailEnum`, `Optional` from typing. Same broad upstreams + PremortemTask. + +- `report.py`: `ReportGenerator` from `worker_plan_internal.report.report_generator`. Also imports `REPORT_EXECUTE_PLAN_SECTION_HIDDEN` from `worker_plan_internal.plan.run_plan_pipeline`. Upstreams: SetupTask, RedlineGateTask, PremiseAttackTask, StrategicDecisionsMarkdownTask, ScenariosMarkdownTask, ConsolidateAssumptionsMarkdownTask, TeamMarkdownTask, RelatedResourcesTask, ConsolidateGovernanceTask, SWOTAnalysisTask, ConvertPitchToMarkdownTask, DataCollectionTask, MarkdownWithDocumentsToCreateAndFindTask, CreateWBSLevel1Task, WBSProjectLevel1AndLevel2AndLevel3Task, ExpertReviewTask, ProjectPlanTask, ReviewPlanTask, ExecutiveSummaryTask, CreateScheduleTask, QuestionsAndAnswersTask, PremortemTask, SelfAuditTask. + +- [ ] **Step 1: Create all 7 files** + +- [ ] **Step 2: Verify and commit** + +```bash +cd worker_plan && python -c "from worker_plan_internal.plan.stages.report import ReportTask; print('OK')" +git add worker_plan/worker_plan_internal/plan/stages/ +git commit -m "refactor: extract Phase 12-13 pipeline stages (schedule, review & report)" +``` + +--- + +### Task 9: Create FullPlanPipeline and slim run_plan_pipeline.py + +**Files:** +- Create: `worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py` +- Modify: `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` + +- [ ] **Step 1: Create `stages/full_plan_pipeline.py`** + +This file imports ALL task classes from the stages and defines `FullPlanPipeline`: + +```python +"""Pipeline orchestrator: declares all stages as Luigi dependencies.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum + +# Phase 1-2: Input & Validation, Purpose +from worker_plan_internal.plan.stages.start_time import StartTimeTask +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.redline_gate import RedlineGateTask +from worker_plan_internal.plan.stages.premise_attack import PremiseAttackTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask + +# Phase 3: Strategic Options & Scenarios +from worker_plan_internal.plan.stages.potential_levers import PotentialLeversTask +from worker_plan_internal.plan.stages.deduplicate_levers import DeduplicateLeversTask +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask +from worker_plan_internal.plan.stages.select_scenario import SelectScenarioTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask + +# Phase 4-5: Context & Assumptions +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask +from worker_plan_internal.plan.stages.review_assumptions import ReviewAssumptionsTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask + +# Phase 6-7: Plan Foundation & Governance +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask +from worker_plan_internal.plan.stages.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgressTask +from worker_plan_internal.plan.stages.governance_phase6_extra import GovernancePhase6ExtraTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask + +# Phase 8: Team +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.find_team_members import FindTeamMembersTask +from worker_plan_internal.plan.stages.enrich_team_contract_type import EnrichTeamMembersWithContractTypeTask +from worker_plan_internal.plan.stages.enrich_team_background_story import EnrichTeamMembersWithBackgroundStoryTask +from worker_plan_internal.plan.stages.enrich_team_environment_info import EnrichTeamMembersWithEnvironmentInfoTask +from worker_plan_internal.plan.stages.review_team import ReviewTeamTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask + +# Phase 9-10: Analysis & Documents +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.identify_documents import IdentifyDocumentsTask +from worker_plan_internal.plan.stages.filter_documents_to_find import FilterDocumentsToFindTask +from worker_plan_internal.plan.stages.filter_documents_to_create import FilterDocumentsToCreateTask +from worker_plan_internal.plan.stages.draft_documents_to_find import DraftDocumentsToFindTask +from worker_plan_internal.plan.stages.draft_documents_to_create import DraftDocumentsToCreateTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask + +# Phase 11: WBS & Pitch +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.create_wbs_level2 import CreateWBSLevel2Task +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task +from worker_plan_internal.plan.stages.create_pitch import CreatePitchTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.identify_task_dependencies import IdentifyTaskDependenciesTask +from worker_plan_internal.plan.stages.estimate_task_durations import EstimateTaskDurationsTask +from worker_plan_internal.plan.stages.create_wbs_level3 import CreateWBSLevel3Task +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task + +# Phase 12-13: Schedule, Final Review & Report +from worker_plan_internal.plan.stages.create_schedule import CreateScheduleTask +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask +from worker_plan_internal.plan.stages.executive_summary import ExecutiveSummaryTask +from worker_plan_internal.plan.stages.questions_and_answers import QuestionsAndAnswersTask +from worker_plan_internal.plan.stages.premortem import PremortemTask +from worker_plan_internal.plan.stages.self_audit import SelfAuditTask +from worker_plan_internal.plan.stages.report import ReportTask +``` + +Copy the `FullPlanPipeline` class body verbatim from SOURCE lines 3776-3848. The `requires()` dict references all the task classes imported above. + +- [ ] **Step 2: Slim `run_plan_pipeline.py`** + +Remove: +1. All task class definitions (lines 217-3848) — everything from `class StartTimeTask` through `class FullPlanPipeline` including its methods. +2. All imports that were ONLY used by the removed task classes. Keep imports used by `PlanTask`, `ExecutePipeline`, `configure_logging`, and the `__main__` block. + +The remaining imports in `run_plan_pipeline.py` should be: + +```python +from dataclasses import dataclass, field +from datetime import date, datetime +import os +import logging +import json +import re +from typing import Any, Optional +import luigi +from pathlib import Path +import sys +from llama_index.core.llms.llm import LLM +from worker_plan_api.filenames import FilenameEnum, ExtraFilenameEnum +from worker_plan_api.pipeline_version import PIPELINE_VERSION +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_internal.utils.planexe_llmconfig import PlanExeLLMConfig +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, LLMModelFromName, ShouldStopCallbackParameters, PipelineStopRequested, RetryConfig +from worker_plan_internal.llm_factory import get_llm_names_by_priority, SPECIAL_AUTO_ID, is_valid_llm_name +from worker_plan_api.model_profile import ModelProfileEnum, normalize_model_profile +from worker_plan_internal.luigi_util.obtain_output_files import ObtainOutputFiles +from worker_plan_internal.plan.pipeline_environment import PipelineEnvironment +from worker_plan_internal.plan.ping_llm import run_ping_llm_report +``` + +- [ ] **Step 3: Update `ExecutePipeline.setup()` to use deferred import** + +Change the `setup()` method in `ExecutePipeline` (around line 3902 in the original). Replace the direct reference to `FullPlanPipeline` with a deferred import: + +```python +def setup(self) -> None: + # ... existing validation code stays the same ... + + from worker_plan_internal.plan.stages.full_plan_pipeline import FullPlanPipeline + full_plan_pipeline_task = FullPlanPipeline( + run_id_dir=self.run_id_dir, + speedvsdetail=self.speedvsdetail, + llm_models=self.llm_models, + _pipeline_executor_callback=self.callback_run_task + ) + self.full_plan_pipeline_task = full_plan_pipeline_task + # ... rest of setup() stays the same ... +``` + +Also update the type hint on `full_plan_pipeline_task` field from `Optional[FullPlanPipeline]` to `Optional[Any]` (since `FullPlanPipeline` is no longer importable at module level): + +```python +full_plan_pipeline_task: Optional[Any] = field(default=None) +``` + +- [ ] **Step 4: Verify the pipeline still works** + +```bash +cd worker_plan && python -c " +from worker_plan_internal.plan.run_plan_pipeline import ExecutePipeline, PlanTask, PipelineProgress, HandleTaskCompletionParameters, _task_class_to_step_label +print('All public symbols importable: OK') +print(_task_class_to_step_label('SWOTAnalysisTask')) +" +``` + +Expected: +``` +All public symbols importable: OK +SWOT Analysis +``` + +- [ ] **Step 5: Run existing tests** + +```bash +cd worker_plan && python -m pytest worker_plan_internal/plan/tests/test_step_label.py -v +``` + +Expected: All tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py +git add worker_plan/worker_plan_internal/plan/run_plan_pipeline.py +git commit -m "refactor: slim run_plan_pipeline.py to framework-only, wire FullPlanPipeline from stages" +``` + +--- + +### Task 10: Update AGENTS.md and remediation roadmap + +**Files:** +- Modify: `worker_plan/AGENTS.md` +- Modify: `docs/proposals/131-codebase-cleanliness-remediation-roadmap.md` + +- [ ] **Step 1: Update `worker_plan/AGENTS.md`** + +Add a new section after the existing guidelines: + +```markdown +## Pipeline Stages (`worker_plan_internal/plan/stages/`) + +Each Luigi pipeline task lives in its own file under `stages/`. This enables: +- Multiple agents working on different stages without merge conflicts +- `self_improve/` targeting individual step files +- Easy DAG insertion (create new file, update downstream `requires()`) + +### Convention for new stages + +1. Create `stages/.py` with one task class +2. Import `PlanTask` from `worker_plan_internal.plan.run_plan_pipeline` +3. Import upstream task dependencies from sibling stage files +4. Declare dependencies via `requires()` returning upstream task(s) +5. Add the new task to `stages/full_plan_pipeline.py`'s `requires()` dict + +### Framework location + +`run_plan_pipeline.py` contains only shared framework: +- `PlanTask` (base class for all stages) +- `ExecutePipeline`, `HandleTaskCompletionParameters`, `PipelineProgress` +- `_task_class_to_step_label`, `configure_logging` +- `__main__` entry point +``` + +- [ ] **Step 2: Update remediation roadmap** + +In `docs/proposals/131-codebase-cleanliness-remediation-roadmap.md`, mark Issue 1 fix step 3 as done: + +Change: +``` +3. Convert `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` from a giant task registry file into a thin pipeline assembly module plus task-specific modules grouped by stage. +``` + +To: +``` +3. ~~Convert `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` from a giant task registry file into a thin pipeline assembly module plus task-specific modules grouped by stage.~~ **Done**: Split 4,257-line monolith into ~66 individual stage files under `stages/` + framework-only core module. +``` + +- [ ] **Step 3: Commit** + +```bash +git add worker_plan/AGENTS.md docs/proposals/131-codebase-cleanliness-remediation-roadmap.md +git commit -m "docs: update AGENTS.md and remediation roadmap for pipeline split" +``` + +--- + +### Task 11: Final verification + +- [ ] **Step 1: Run all existing tests** + +```bash +cd worker_plan && python -m pytest worker_plan_internal/plan/tests/ -v +``` + +Expected: All tests pass. + +- [ ] **Step 2: Verify import chain** + +```bash +cd worker_plan && python -c " +from worker_plan_internal.plan.stages.full_plan_pipeline import FullPlanPipeline +from worker_plan_internal.plan.run_plan_pipeline import ExecutePipeline +print(f'FullPlanPipeline requires {len(FullPlanPipeline(run_id_dir=\"/tmp/test\").requires())} tasks') +print('Import chain: OK') +" +``` + +- [ ] **Step 3: Count lines in slimmed run_plan_pipeline.py** + +```bash +wc -l worker_plan/worker_plan_internal/plan/run_plan_pipeline.py +``` + +Expected: ~280 lines (down from 4,257). + +- [ ] **Step 4: Count stage files** + +```bash +ls worker_plan/worker_plan_internal/plan/stages/*.py | wc -l +``` + +Expected: ~67 files (66 stages + `__init__.py` + `full_plan_pipeline.py`). diff --git a/docs/superpowers/specs/2026-04-02-split-run-plan-pipeline-design.md b/docs/superpowers/specs/2026-04-02-split-run-plan-pipeline-design.md new file mode 100644 index 000000000..11d005920 --- /dev/null +++ b/docs/superpowers/specs/2026-04-02-split-run-plan-pipeline-design.md @@ -0,0 +1,233 @@ +# Split run_plan_pipeline.py — Design Spec + +**Date:** 2026-04-02 +**Status:** Approved +**Tags:** `worker_plan`, `refactor`, `maintainability`, `agent-isolation` + +--- + +## Goal + +Split `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` (4,257 lines) into ~66 individual stage files plus a slim core module, optimizing for parallel agent work, `self_improve/` integration, and conflict-free DAG insertion. + +## Problem + +`run_plan_pipeline.py` contains ~66 Luigi task classes, the `PlanTask` base class, `ExecutePipeline`, `FullPlanPipeline`, and supporting utilities — all in one file. This means: + +1. Two agents working on different pipeline steps must edit the same file, causing merge conflicts. +2. `self_improve/` cannot target an individual step file — it must reason about a 4,257-line module. +3. Inserting a new task in the DAG requires editing the giant file, risking unrelated merge conflicts with other in-flight work. + +## Design Priorities + +These priorities (from the user) override backward-compatibility concerns: + +1. **Agent isolation** — each pipeline step in its own file so agents never conflict. +2. **Easy DAG insertion** — adding a new task between two existing tasks touches only 2 files (the new file and the downstream file's `requires()`), never a central registry. +3. **`self_improve/` addressability** — each step is an individually importable module. + +## Approach + +Extract each task class into its own file under a new `stages/` directory. Each file declares its own Luigi `requires()` dependencies by importing its upstream task(s). `run_plan_pipeline.py` keeps only the shared framework (`PlanTask`, `ExecutePipeline`, etc.). + +## New Module Layout + +### `run_plan_pipeline.py` (~280 lines, slimmed core) + +Keeps: +- All imports needed by the framework classes +- `PlanTask` base class (lines 105–215) +- `_task_class_to_step_label` utility (lines 3851–3862) +- `PipelineProgress` dataclass (lines 3866–3871) +- `HandleTaskCompletionParameters` dataclass (lines 3875–3878) +- `ExecutePipeline` dataclass (lines 3882–4107) +- `DemoStoppingExecutePipeline` (lines 4109–4116) +- `configure_logging` (lines 4119–4148) +- `__main__` block (lines 4152–4258) +- Module-level constants: `logger`, `DEFAULT_LLM_MODEL`, `REPORT_EXECUTE_PLAN_SECTION_HIDDEN` + +Changes: +- `ExecutePipeline.setup()` imports `FullPlanPipeline` from `stages.full_plan_pipeline` instead of referencing it as a module-level class. +- The `__main__` block's `DemoStoppingExecutePipeline` reference stays (it's in the same file). +- All ~66 task class definitions and their imports are removed. + +### `stages/` directory + +Each file contains exactly one task class (except where noted). Files follow this pattern: + +```python +"""Pipeline stage: .""" +import logging +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum +# ... executor imports specific to this stage +# ... upstream task imports from sibling stage files + +logger = logging.getLogger(__name__) + +class FooTask(PlanTask): + def requires(self): + return self.clone(UpstreamTask) + + def output(self): + return self.local_target(FilenameEnum.FOO) + + def run_with_llm(self, llm): + # ... verbatim from current code +``` + +### `stages/__init__.py` + +Empty file. Stage files are imported directly by whoever needs them (other stages, `FullPlanPipeline`). + +### `stages/full_plan_pipeline.py` + +Contains `FullPlanPipeline` which imports all task classes and lists them in `requires()`. This is the only file that imports every stage — it's the DAG root. + +**Note:** `FullPlanPipeline.requires()` currently lists ALL tasks explicitly (not just leaves). This is preserved verbatim to maintain identical Luigi behavior. Optimizing to leaf-only listing is a separate future change. + +### Complete file list + +``` +stages/ +├── __init__.py +├── full_plan_pipeline.py +│ +│ # Phase 1: Input & Validation +├── start_time.py (StartTimeTask) +├── setup.py (SetupTask) +├── redline_gate.py (RedlineGateTask) +├── premise_attack.py (PremiseAttackTask) +│ +│ # Phase 2: Purpose & Classification +├── identify_purpose.py (IdentifyPurposeTask) +├── plan_type.py (PlanTypeTask) +│ +│ # Phase 3: Strategic Options & Scenarios +├── potential_levers.py (PotentialLeversTask) +├── deduplicate_levers.py (DeduplicateLeversTask) +├── enrich_levers.py (EnrichLeversTask) +├── focus_on_vital_few_levers.py (FocusOnVitalFewLeversTask) +├── strategic_decisions_markdown.py (StrategicDecisionsMarkdownTask) +├── candidate_scenarios.py (CandidateScenariosTask) +├── select_scenario.py (SelectScenarioTask) +├── scenarios_markdown.py (ScenariosMarkdownTask) +│ +│ # Phase 4: Context +├── physical_locations.py (PhysicalLocationsTask) +├── currency_strategy.py (CurrencyStrategyTask) +├── identify_risks.py (IdentifyRisksTask) +│ +│ # Phase 5: Assumptions +├── make_assumptions.py (MakeAssumptionsTask) +├── distill_assumptions.py (DistillAssumptionsTask) +├── review_assumptions.py (ReviewAssumptionsTask) +├── consolidate_assumptions_markdown.py (ConsolidateAssumptionsMarkdownTask) +│ +│ # Phase 6: Plan Foundation +├── pre_project_assessment.py (PreProjectAssessmentTask) +├── project_plan.py (ProjectPlanTask) +│ +│ # Phase 7: Governance +├── governance_phase1_audit.py (GovernancePhase1AuditTask) +├── governance_phase2_bodies.py (GovernancePhase2BodiesTask) +├── governance_phase3_impl_plan.py (GovernancePhase3ImplPlanTask) +├── governance_phase4_decision_escalation_matrix.py (GovernancePhase4DecisionEscalationMatrixTask) +├── governance_phase5_monitoring_progress.py (GovernancePhase5MonitoringProgressTask) +├── governance_phase6_extra.py (GovernancePhase6ExtraTask) +├── consolidate_governance.py (ConsolidateGovernanceTask) +│ +│ # Phase 8: Team +├── related_resources.py (RelatedResourcesTask) +├── find_team_members.py (FindTeamMembersTask) +├── enrich_team_contract_type.py (EnrichTeamMembersWithContractTypeTask) +├── enrich_team_background_story.py (EnrichTeamMembersWithBackgroundStoryTask) +├── enrich_team_environment_info.py (EnrichTeamMembersWithEnvironmentInfoTask) +├── review_team.py (ReviewTeamTask) +├── team_markdown.py (TeamMarkdownTask) +│ +│ # Phase 9: Analysis +├── swot_analysis.py (SWOTAnalysisTask) +├── expert_review.py (ExpertReviewTask) +│ +│ # Phase 10: Documents +├── data_collection.py (DataCollectionTask) +├── identify_documents.py (IdentifyDocumentsTask) +├── filter_documents_to_find.py (FilterDocumentsToFindTask) +├── filter_documents_to_create.py (FilterDocumentsToCreateTask) +├── draft_documents_to_find.py (DraftDocumentsToFindTask) +├── draft_documents_to_create.py (DraftDocumentsToCreateTask) +├── markdown_documents.py (MarkdownWithDocumentsToCreateAndFindTask) +│ +│ # Phase 11: WBS & Pitch +├── create_wbs_level1.py (CreateWBSLevel1Task) +├── create_wbs_level2.py (CreateWBSLevel2Task) +├── wbs_project_level1_and_level2.py (WBSProjectLevel1AndLevel2Task) +├── create_pitch.py (CreatePitchTask) +├── convert_pitch_to_markdown.py (ConvertPitchToMarkdownTask) +├── identify_task_dependencies.py (IdentifyTaskDependenciesTask) +├── estimate_task_durations.py (EstimateTaskDurationsTask) +├── create_wbs_level3.py (CreateWBSLevel3Task) +├── wbs_project_level1_level2_level3.py (WBSProjectLevel1AndLevel2AndLevel3Task) +│ +│ # Phase 12: Schedule & Final Review +├── create_schedule.py (CreateScheduleTask) +├── review_plan.py (ReviewPlanTask) +├── executive_summary.py (ExecutiveSummaryTask) +├── questions_and_answers.py (QuestionsAndAnswersTask) +├── premortem.py (PremortemTask) +├── self_audit.py (SelfAuditTask) +│ +│ # Phase 13: Report +└── report.py (ReportTask) +``` + +## DAG Insertion Example + +To insert a new `ValidateBudgetTask` between `CurrencyStrategyTask` and `IdentifyRisksTask`: + +1. Create `stages/validate_budget.py`: + ```python + from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask + + class ValidateBudgetTask(PlanTask): + def requires(self): + return self.clone(CurrencyStrategyTask) + # ... + ``` + +2. Edit `stages/identify_risks.py` — change its `requires()` to depend on `ValidateBudgetTask` instead of `CurrencyStrategyTask`. + +3. Add one line to `stages/full_plan_pipeline.py`'s `requires()` dict. + +Only 3 files touched. No other stage files are affected. + +## What Does NOT Change + +- No behavioral changes — identical DAG, identical task execution, identical output files +- No new dependencies +- No changes to Luigi task parameters, output filenames, or callback mechanisms +- No changes to `worker_plan_database/app.py` (imports `ExecutePipeline` from `run_plan_pipeline`, which stays) +- No changes to `test_step_label.py` (imports `_task_class_to_step_label` from `run_plan_pipeline`, which stays) +- The `__main__` entry point still works: `python -m worker_plan_internal.plan.run_plan_pipeline` + +## Code Movement Rules + +1. **Verbatim extraction** — task class bodies are moved exactly as-is, no logic changes. +2. **Each file gets only the imports it needs** — no bulk import block copied to every file. +3. **The `REPORT_EXECUTE_PLAN_SECTION_HIDDEN` constant** stays in `run_plan_pipeline.py` since it's used by `ReportTask`. `stages/report.py` imports it from there. + +## AGENTS.md Update + +`worker_plan/AGENTS.md` will be updated to document: +- The `stages/` directory and the one-file-per-task convention +- How to add a new pipeline stage (create file, wire `requires()`, add to `FullPlanPipeline`) +- That `run_plan_pipeline.py` contains only shared framework, not task implementations + +## Risks + +1. **Circular imports**: `stages/*.py` files import `PlanTask` from `run_plan_pipeline.py`. `run_plan_pipeline.py` imports `FullPlanPipeline` from `stages/full_plan_pipeline.py` (only inside `ExecutePipeline.setup()`, not at module level). No circularity because the cross-import is deferred to method call time. + +2. **Luigi task discovery**: Luigi resolves tasks via `requires()` chains starting from `FullPlanPipeline`. Since each stage file imports its upstream tasks, all task classes are loaded when `FullPlanPipeline` is imported. No registration mechanism needed. + +3. **`self_improve/` runner compatibility**: The runner imports and executes step source files directly. After the split, individual step executor classes (like `IdentifyPotentialLevers`) still live in their original locations (`worker_plan_internal/lever/`). The Luigi task wrappers move to `stages/`, but `self_improve/` doesn't import those — it imports the executor classes directly. No changes needed. From b139b7ea80deeaeb9f502b7a27212dca3115767a Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:30:21 +0200 Subject: [PATCH 02/12] refactor: extract Phase 1-2 pipeline stages to individual files Extract StartTimeTask, SetupTask, RedlineGateTask, PremiseAttackTask, IdentifyPurposeTask, PlanTypeTask into stages/ directory. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/__init__.py | 0 .../plan/stages/identify_purpose.py | 33 ++++++++++++++ .../plan/stages/plan_type.py | 44 +++++++++++++++++++ .../plan/stages/premise_attack.py | 32 ++++++++++++++ .../plan/stages/redline_gate.py | 30 +++++++++++++ .../worker_plan_internal/plan/stages/setup.py | 14 ++++++ .../plan/stages/start_time.py | 14 ++++++ 7 files changed, 167 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/__init__.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/identify_purpose.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/plan_type.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/premise_attack.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/redline_gate.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/setup.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/start_time.py diff --git a/worker_plan/worker_plan_internal/plan/stages/__init__.py b/worker_plan/worker_plan_internal/plan/stages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/worker_plan/worker_plan_internal/plan/stages/identify_purpose.py b/worker_plan/worker_plan_internal/plan/stages/identify_purpose.py new file mode 100644 index 000000000..6f7bdd673 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/identify_purpose.py @@ -0,0 +1,33 @@ +"""IdentifyPurposeTask - Determine if this is a business/personal/other plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_purpose import IdentifyPurpose +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask + + +class IdentifyPurposeTask(PlanTask): + """ + Determine if this is this going to be a business/personal/other plan. + """ + def requires(self): + return self.clone(SetupTask) + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.IDENTIFY_PURPOSE_RAW), + 'markdown': self.local_target(FilenameEnum.IDENTIFY_PURPOSE_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input().open("r") as f: + plan_prompt = f.read() + + identify_purpose = IdentifyPurpose.execute(llm, plan_prompt) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + identify_purpose.save_raw(output_raw_path) + output_markdown_path = self.output()['markdown'].path + identify_purpose.save_markdown(output_markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/plan_type.py b/worker_plan/worker_plan_internal/plan/stages/plan_type.py new file mode 100644 index 000000000..af6d24234 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/plan_type.py @@ -0,0 +1,44 @@ +"""PlanTypeTask - Determine if the plan is purely digital or requires physical locations.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_plan_type import IdentifyPlanType +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask + + +class PlanTypeTask(PlanTask): + """ + Determine if the plan is purely digital or requires physical locations. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PLAN_TYPE_RAW), + 'markdown': self.local_target(FilenameEnum.PLAN_TYPE_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}" + ) + + identify_plan_type = IdentifyPlanType.execute(llm, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + identify_plan_type.save_raw(str(output_raw_path)) + output_markdown_path = self.output()['markdown'].path + identify_plan_type.save_markdown(str(output_markdown_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/premise_attack.py b/worker_plan/worker_plan_internal/plan/stages/premise_attack.py new file mode 100644 index 000000000..7f42939bb --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/premise_attack.py @@ -0,0 +1,32 @@ +"""PremiseAttackTask - Attacks the premises of the plan prompt.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.diagnostics.premise_attack import PremiseAttack +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask + + +class PremiseAttackTask(PlanTask): + def requires(self): + return self.clone(SetupTask) + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PREMISE_ATTACK_RAW), + 'markdown': self.local_target(FilenameEnum.PREMISE_ATTACK_MARKDOWN) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input().open("r") as f: + plan_prompt = f.read() + + premise_attack = PremiseAttack.execute(llm_executor, plan_prompt) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + premise_attack.save_raw(output_raw_path) + output_markdown_path = self.output()['markdown'].path + premise_attack.save_markdown(output_markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/redline_gate.py b/worker_plan/worker_plan_internal/plan/stages/redline_gate.py new file mode 100644 index 000000000..401086514 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/redline_gate.py @@ -0,0 +1,30 @@ +"""RedlineGateTask - Checks the plan prompt against redline criteria.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.diagnostics.redline_gate import RedlineGate +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask + + +class RedlineGateTask(PlanTask): + def requires(self): + return self.clone(SetupTask) + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.REDLINE_GATE_RAW), + 'markdown': self.local_target(FilenameEnum.REDLINE_GATE_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input().open("r") as f: + plan_prompt = f.read() + + redline_gate = RedlineGate.execute(llm, plan_prompt) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + redline_gate.save_raw(output_raw_path) + output_markdown_path = self.output()['markdown'].path + redline_gate.save_markdown(output_markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/setup.py b/worker_plan/worker_plan_internal/plan/stages/setup.py new file mode 100644 index 000000000..bd0ba06d6 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/setup.py @@ -0,0 +1,14 @@ +"""SetupTask - The plan prompt text provided by the user.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum + + +class SetupTask(PlanTask): + """The plan prompt text provided by the user.""" + def output(self): + return self.local_target(FilenameEnum.INITIAL_PLAN) + + def run(self): + # The Gradio/Flask app that starts the luigi pipeline, must first create the `INITIAL_PLAN` file inside the `run_id_dir`. + # This code will ONLY run if the Gradio/Flask app *failed* to create the file. + raise AssertionError(f"This code is not supposed to be run. Before starting the pipeline the '{FilenameEnum.INITIAL_PLAN.value}' file must be present in the `run_id_dir`: {self.run_id_dir!r}") diff --git a/worker_plan/worker_plan_internal/plan/stages/start_time.py b/worker_plan/worker_plan_internal/plan/stages/start_time.py new file mode 100644 index 000000000..c655d279b --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/start_time.py @@ -0,0 +1,14 @@ +"""StartTimeTask - The timestamp when the pipeline was started.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum + + +class StartTimeTask(PlanTask): + """The timestamp when the pipeline was started.""" + def output(self): + return self.local_target(FilenameEnum.START_TIME) + + def run(self): + # The Gradio/Flask app that starts the luigi pipeline, must first create the `START_TIME` file inside the `run_id_dir`. + # This code will ONLY run if the Gradio/Flask app *failed* to create the file. + raise AssertionError(f"This code is not supposed to be run. Before starting the pipeline the '{FilenameEnum.START_TIME.value}' file must be present in the `run_id_dir`: {self.run_id_dir!r}") From bd267231f2742ea13bcbd6a26bdd3d93d639bb6e Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:32:58 +0200 Subject: [PATCH 03/12] refactor: extract Phase 3 pipeline stages (levers & scenarios) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/candidate_scenarios.py | 60 +++++++++++++++++ .../plan/stages/deduplicate_levers.py | 57 ++++++++++++++++ .../plan/stages/enrich_levers.py | 58 ++++++++++++++++ .../plan/stages/focus_on_vital_few_levers.py | 57 ++++++++++++++++ .../plan/stages/potential_levers.py | 51 ++++++++++++++ .../plan/stages/scenarios_markdown.py | 37 ++++++++++ .../plan/stages/select_scenario.py | 67 +++++++++++++++++++ .../stages/strategic_decisions_markdown.py | 35 ++++++++++ 8 files changed, 422 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/candidate_scenarios.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/deduplicate_levers.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/enrich_levers.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/focus_on_vital_few_levers.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/potential_levers.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/scenarios_markdown.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/select_scenario.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/strategic_decisions_markdown.py diff --git a/worker_plan/worker_plan_internal/plan/stages/candidate_scenarios.py b/worker_plan/worker_plan_internal/plan/stages/candidate_scenarios.py new file mode 100644 index 000000000..958f7a48e --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/candidate_scenarios.py @@ -0,0 +1,60 @@ +"""CandidateScenariosTask - Combinations of the vital few levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.candidate_scenarios import CandidateScenarios +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask + + +class CandidateScenariosTask(PlanTask): + """ + Combinations of the vital few levers. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.CANDIDATE_SCENARIOS_RAW), + 'clean': self.local_target(FilenameEnum.CANDIDATE_SCENARIOS_CLEAN) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['levers_vital_few']['raw'].open("r") as f: + lever_item_list = json.load(f)["levers"] + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + ) + + scenarios = CandidateScenarios.execute( + llm_executor=llm_executor, + project_context=query, + raw_vital_levers=lever_item_list + ) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + scenarios.save_raw(str(output_raw_path)) + output_clean_path = self.output()['clean'].path + scenarios.save_clean(str(output_clean_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/deduplicate_levers.py b/worker_plan/worker_plan_internal/plan/stages/deduplicate_levers.py new file mode 100644 index 000000000..82f6b645e --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/deduplicate_levers.py @@ -0,0 +1,57 @@ +"""DeduplicateLeversTask - The potential levers usually have some redundant levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.deduplicate_levers import DeduplicateLevers +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.potential_levers import PotentialLeversTask + + +class DeduplicateLeversTask(PlanTask): + """ + The potential levers usually have some redundant levers. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'potential_levers': self.clone(PotentialLeversTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.DEDUPLICATED_LEVERS_RAW) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['potential_levers']['clean'].open("r") as f: + lever_item_list = json.load(f) + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}" + ) + + deduplicate_levers = DeduplicateLevers.execute( + llm_executor, + project_context=query, + raw_levers_list=lever_item_list + ) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + deduplicate_levers.save_raw(str(output_raw_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/enrich_levers.py b/worker_plan/worker_plan_internal/plan/stages/enrich_levers.py new file mode 100644 index 000000000..179b66a8d --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/enrich_levers.py @@ -0,0 +1,58 @@ +"""EnrichLeversTask - Enrich potential levers with more information.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.enrich_potential_levers import EnrichPotentialLevers +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.deduplicate_levers import DeduplicateLeversTask + + +class EnrichLeversTask(PlanTask): + """ + Enrich potential levers with more information. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'deduplicate_levers': self.clone(DeduplicateLeversTask), + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.ENRICHED_LEVERS_RAW) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['deduplicate_levers']['raw'].open("r") as f: + json_dict = json.load(f) + lever_item_list = json_dict["deduplicated_levers"] + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}" + ) + + enrich_potential_levers = EnrichPotentialLevers.execute( + llm_executor, + project_context=query, + raw_levers_list=lever_item_list + ) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + enrich_potential_levers.save_raw(str(output_raw_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/focus_on_vital_few_levers.py b/worker_plan/worker_plan_internal/plan/stages/focus_on_vital_few_levers.py new file mode 100644 index 000000000..83ed59cf5 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/focus_on_vital_few_levers.py @@ -0,0 +1,57 @@ +"""FocusOnVitalFewLeversTask - Apply the 80/20 principle to the levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.focus_on_vital_few_levers import FocusOnVitalFewLevers +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask + + +class FocusOnVitalFewLeversTask(PlanTask): + """ + Apply the 80/20 principle to the levers. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'enriched_levers': self.clone(EnrichLeversTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.VITAL_FEW_LEVERS_RAW) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['enriched_levers']['raw'].open("r") as f: + lever_item_list = json.load(f)["characterized_levers"] + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + ) + + focus_on_vital_few_levers = FocusOnVitalFewLevers.execute( + llm_executor, + project_context=query, + raw_levers_list=lever_item_list + ) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + focus_on_vital_few_levers.save_raw(str(output_raw_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/potential_levers.py b/worker_plan/worker_plan_internal/plan/stages/potential_levers.py new file mode 100644 index 000000000..2f24830ec --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/potential_levers.py @@ -0,0 +1,51 @@ +"""PotentialLeversTask - Identify potential levers that can be adjusted.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.identify_potential_levers import IdentifyPotentialLevers +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask + + +class PotentialLeversTask(PlanTask): + """ + Identify potential levers that can be adjusted. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.POTENTIAL_LEVERS_RAW), + 'clean': self.local_target(FilenameEnum.POTENTIAL_LEVERS_CLEAN), + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}" + ) + + identify_potential_levers = IdentifyPotentialLevers.execute(llm_executor, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + identify_potential_levers.save_raw(str(output_raw_path)) + output_clean_path = self.output()['clean'].path + identify_potential_levers.save_clean(str(output_clean_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/scenarios_markdown.py b/worker_plan/worker_plan_internal/plan/stages/scenarios_markdown.py new file mode 100644 index 000000000..fa707435b --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/scenarios_markdown.py @@ -0,0 +1,37 @@ +"""ScenariosMarkdownTask - Present the scenarios in a human readable format.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.scenarios_markdown import ScenariosMarkdown +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask +from worker_plan_internal.plan.stages.select_scenario import SelectScenarioTask + + +class ScenariosMarkdownTask(PlanTask): + """ + Present the scenarios in a human readable format. + """ + def requires(self): + return { + 'candidate_scenarios': self.clone(CandidateScenariosTask), + 'selected_scenario': self.clone(SelectScenarioTask) + } + + def output(self): + return { + 'markdown': self.local_target(FilenameEnum.SCENARIOS_MARKDOWN) + } + + def run(self): + with self.input()['candidate_scenarios']['clean'].open("r") as f: + scenarios_list = json.load(f).get('scenarios', []) + with self.input()['selected_scenario']['clean'].open("r") as f: + selected_scenario_dict = json.load(f) + + # Extract the required data from the selected scenario + plan_characteristics = selected_scenario_dict.get('plan_characteristics', {}) + scenario_assessments = selected_scenario_dict.get('scenario_assessments', []) + final_choice = selected_scenario_dict.get('final_choice', {}) + + result = ScenariosMarkdown(scenarios_list, plan_characteristics, scenario_assessments, final_choice) + result.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/select_scenario.py b/worker_plan/worker_plan_internal/plan/stages/select_scenario.py new file mode 100644 index 000000000..ef5a9bac3 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/select_scenario.py @@ -0,0 +1,67 @@ +"""SelectScenarioTask - Pick the best fitting scenario to make a plan for.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.select_scenario import SelectScenario +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask + + +class SelectScenarioTask(PlanTask): + """ + Pick the best fitting scenario to make a plan for. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask), + 'candidate_scenarios': self.clone(CandidateScenariosTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.SELECTED_SCENARIO_RAW), + 'clean': self.local_target(FilenameEnum.SELECTED_SCENARIO_CLEAN) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['levers_vital_few']['raw'].open("r") as f: + lever_item_list = json.load(f)["levers"] + with self.input()['candidate_scenarios']['clean'].open("r") as f: + scenarios_list = json.load(f).get('scenarios', []) + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + f"File 'levers_vital_few.json':\n{format_json_for_use_in_query(lever_item_list)}\n\n" + f"File 'candidate_scenarios.json':\n{format_json_for_use_in_query(scenarios_list)}" + ) + + select_scenario = SelectScenario.execute( + llm_executor=llm_executor, + project_context=query, + scenarios=scenarios_list + ) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + select_scenario.save_raw(str(output_raw_path)) + output_clean_path = self.output()['clean'].path + select_scenario.save_clean(str(output_clean_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/strategic_decisions_markdown.py b/worker_plan/worker_plan_internal/plan/stages/strategic_decisions_markdown.py new file mode 100644 index 000000000..123de49a5 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/strategic_decisions_markdown.py @@ -0,0 +1,35 @@ +"""StrategicDecisionsMarkdownTask - Human readable markdown with the levers.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.lever.strategic_decisions_markdown import StrategicDecisionsMarkdown +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask + + +class StrategicDecisionsMarkdownTask(PlanTask): + """ + Human readable markdown with the levers. + """ + def requires(self): + return { + 'enriched_levers': self.clone(EnrichLeversTask), + 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask) + } + + def output(self): + return { + 'markdown': self.local_target(FilenameEnum.STRATEGIC_DECISIONS_MARKDOWN) + } + + def run(self): + with self.input()['enriched_levers']['raw'].open("r") as f: + enrich_lever_list = json.load(f)["characterized_levers"] + with self.input()['levers_vital_few']['raw'].open("r") as f: + vital_data = json.load(f) + vital_lever_list = vital_data["levers"] + lever_assessments_list = vital_data.get("response", {}).get("lever_assessments", []) + vital_levers_summary = vital_data.get("response", {}).get("summary", "") + + result = StrategicDecisionsMarkdown(enrich_lever_list, vital_lever_list, vital_levers_summary, lever_assessments_list) + result.save_markdown(self.output()['markdown'].path) From b84fb7e7da4b5ec1dd0cbc122cf1e727d4f34f5c Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:50:58 +0200 Subject: [PATCH 04/12] refactor: extract Phase 4-5 pipeline stages (context & assumptions) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../consolidate_assumptions_markdown.py | 102 ++++++++++++++++++ .../plan/stages/currency_strategy.py | 64 +++++++++++ .../plan/stages/distill_assumptions.py | 62 +++++++++++ .../plan/stages/identify_risks.py | 69 ++++++++++++ .../plan/stages/make_assumptions.py | 77 +++++++++++++ .../plan/stages/physical_locations.py | 80 ++++++++++++++ .../plan/stages/review_assumptions.py | 80 ++++++++++++++ 7 files changed, 534 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/consolidate_assumptions_markdown.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/currency_strategy.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/distill_assumptions.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/identify_risks.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/make_assumptions.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/physical_locations.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/review_assumptions.py diff --git a/worker_plan/worker_plan_internal/plan/stages/consolidate_assumptions_markdown.py b/worker_plan/worker_plan_internal/plan/stages/consolidate_assumptions_markdown.py new file mode 100644 index 000000000..f150618b6 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/consolidate_assumptions_markdown.py @@ -0,0 +1,102 @@ +"""ConsolidateAssumptionsMarkdownTask - Combines multiple small markdown documents into one.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.shorten_markdown import ShortenMarkdown +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask +from worker_plan_internal.plan.stages.review_assumptions import ReviewAssumptionsTask + +logger = logging.getLogger(__name__) + + +class ConsolidateAssumptionsMarkdownTask(PlanTask): + """ + Combines multiple small markdown documents into a single big document. + """ + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'currency_strategy': self.clone(CurrencyStrategyTask), + 'identify_risks': self.clone(IdentifyRisksTask), + 'make_assumptions': self.clone(MakeAssumptionsTask), + 'distill_assumptions': self.clone(DistillAssumptionsTask), + 'review_assumptions': self.clone(ReviewAssumptionsTask) + } + + def output(self): + return { + 'full': self.local_target(FilenameEnum.CONSOLIDATE_ASSUMPTIONS_FULL_MARKDOWN), + 'short': self.local_target(FilenameEnum.CONSOLIDATE_ASSUMPTIONS_SHORT_MARKDOWN) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Define the list of (title, path) tuples + title_path_list = [ + ('Purpose', self.input()['identify_purpose']['markdown'].path), + ('Plan Type', self.input()['plan_type']['markdown'].path), + ('Physical Locations', self.input()['physical_locations']['markdown'].path), + ('Currency Strategy', self.input()['currency_strategy']['markdown'].path), + ('Identify Risks', self.input()['identify_risks']['markdown'].path), + ('Make Assumptions', self.input()['make_assumptions']['markdown'].path), + ('Distill Assumptions', self.input()['distill_assumptions']['markdown'].path), + ('Review Assumptions', self.input()['review_assumptions']['markdown'].path) + ] + + # Read the files and handle exceptions + full_markdown_chunks = [] + short_markdown_chunks = [] + for title, path in title_path_list: + try: + with open(path, 'r', encoding='utf-8') as f: + markdown_chunk = f.read() + full_markdown_chunks.append(f"# {title}\n\n{markdown_chunk}") + except FileNotFoundError: + logger.warning(f"Markdown file not found: {path} (from {title})") + full_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") + short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") + continue + except Exception as e: + logger.error(f"Error reading markdown file {path} (from {title}): {e}") + full_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") + short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") + continue + + # IDEA: If the chunk file already exist, then there is no need to run the LLM again. + def execute_shorten_markdown(llm: LLM) -> ShortenMarkdown: + return ShortenMarkdown.execute(llm, markdown_chunk) + + try: + shorten_markdown = llm_executor.run(execute_shorten_markdown) + short_markdown_chunks.append(f"# {title}\n{shorten_markdown.markdown}") + except PipelineStopRequested: + # Re-raise PipelineStopRequested without wrapping it + raise + except Exception as e: + logger.error(f"Error shortening markdown file {path} (from {title}): {e}") + short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError shortening markdown file.") + continue + + # Combine the markdown chunks + full_markdown = "\n\n".join(full_markdown_chunks) + short_markdown = "\n\n".join(short_markdown_chunks) + + # Write the result to disk. + output_full_markdown_path = self.output()['full'].path + with open(output_full_markdown_path, "w", encoding="utf-8") as f: + f.write(full_markdown) + + output_short_markdown_path = self.output()['short'].path + with open(output_short_markdown_path, "w", encoding="utf-8") as f: + f.write(short_markdown) diff --git a/worker_plan/worker_plan_internal/plan/stages/currency_strategy.py b/worker_plan/worker_plan_internal/plan/stages/currency_strategy.py new file mode 100644 index 000000000..87992af03 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/currency_strategy.py @@ -0,0 +1,64 @@ +"""CurrencyStrategyTask - Identify/suggest what currency to use for the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.currency_strategy import CurrencyStrategy +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask + + +class CurrencyStrategyTask(PlanTask): + """ + Identify/suggest what currency to use for the plan, depending on the physical locations. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.CURRENCY_STRATEGY_RAW), + 'markdown': self.local_target(FilenameEnum.CURRENCY_STRATEGY_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['physical_locations']['markdown'].open("r") as f: + physical_locations_markdown = f.read() + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'physical_locations.md':\n{physical_locations_markdown}" + ) + + currency_strategy = CurrencyStrategy.execute(llm, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + currency_strategy.save_raw(str(output_raw_path)) + output_markdown_path = self.output()['markdown'].path + currency_strategy.save_markdown(str(output_markdown_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/distill_assumptions.py b/worker_plan/worker_plan_internal/plan/stages/distill_assumptions.py new file mode 100644 index 000000000..c645d54f5 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/distill_assumptions.py @@ -0,0 +1,62 @@ +"""DistillAssumptionsTask - Distill raw assumption data.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.distill_assumptions import DistillAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask + + +class DistillAssumptionsTask(PlanTask): + """ + Distill raw assumption data. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'make_assumptions': self.clone(MakeAssumptionsTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.DISTILL_ASSUMPTIONS_RAW), + 'markdown': self.local_target(FilenameEnum.DISTILL_ASSUMPTIONS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + make_assumptions_target = self.input()['make_assumptions']['clean'] + with make_assumptions_target.open("r") as f: + assumptions_raw_data = json.load(f) + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.json':\n{format_json_for_use_in_query(assumptions_raw_data)}" + ) + + distill_assumptions = DistillAssumptions.execute(llm, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + distill_assumptions.save_raw(str(output_raw_path)) + output_markdown_path = self.output()['markdown'].path + distill_assumptions.save_markdown(str(output_markdown_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/identify_risks.py b/worker_plan/worker_plan_internal/plan/stages/identify_risks.py new file mode 100644 index 000000000..df662f0e8 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/identify_risks.py @@ -0,0 +1,69 @@ +"""IdentifyRisksTask - Identify risks for the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.identify_risks import IdentifyRisks +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask + + +class IdentifyRisksTask(PlanTask): + """ + Identify risks for the plan, depending on the physical locations. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'currency_strategy': self.clone(CurrencyStrategyTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.IDENTIFY_RISKS_RAW), + 'markdown': self.local_target(FilenameEnum.IDENTIFY_RISKS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['physical_locations']['markdown'].open("r") as f: + physical_locations_markdown = f.read() + with self.input()['currency_strategy']['markdown'].open("r") as f: + currency_strategy_markdown = f.read() + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'physical_locations.md':\n{physical_locations_markdown}\n\n" + f"File 'currency_strategy.md':\n{currency_strategy_markdown}" + ) + + identify_risks = IdentifyRisks.execute(llm, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + identify_risks.save_raw(str(output_raw_path)) + output_markdown_path = self.output()['markdown'].path + identify_risks.save_markdown(str(output_markdown_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/make_assumptions.py b/worker_plan/worker_plan_internal/plan/stages/make_assumptions.py new file mode 100644 index 000000000..282f27643 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/make_assumptions.py @@ -0,0 +1,77 @@ +"""MakeAssumptionsTask - Make assumptions about the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.make_assumptions import MakeAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask + + +class MakeAssumptionsTask(PlanTask): + """ + Make assumptions about the plan. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'currency_strategy': self.clone(CurrencyStrategyTask), + 'identify_risks': self.clone(IdentifyRisksTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_RAW), + 'clean': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_CLEAN), + 'markdown': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['physical_locations']['markdown'].open("r") as f: + physical_locations_markdown = f.read() + with self.input()['currency_strategy']['markdown'].open("r") as f: + currency_strategy_markdown = f.read() + with self.input()['identify_risks']['markdown'].open("r") as f: + identify_risks_markdown = f.read() + + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'physical_locations.md':\n{physical_locations_markdown}\n\n" + f"File 'currency_strategy.md':\n{currency_strategy_markdown}\n\n" + f"File 'identify_risks.md':\n{identify_risks_markdown}" + ) + + make_assumptions = MakeAssumptions.execute(llm, query) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + make_assumptions.save_raw(str(output_raw_path)) + output_clean_path = self.output()['clean'].path + make_assumptions.save_assumptions(str(output_clean_path)) + output_markdown_path = self.output()['markdown'].path + make_assumptions.save_markdown(str(output_markdown_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/physical_locations.py b/worker_plan/worker_plan_internal/plan/stages/physical_locations.py new file mode 100644 index 000000000..85f74a68d --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/physical_locations.py @@ -0,0 +1,80 @@ +"""PhysicalLocationsTask - Identify/suggest physical locations for the plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.physical_locations import PhysicalLocations +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask + +logger = logging.getLogger(__name__) + + +class PhysicalLocationsTask(PlanTask): + """ + Identify/suggest physical locations for the plan. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PHYSICAL_LOCATIONS_RAW), + 'markdown': self.local_target(FilenameEnum.PHYSICAL_LOCATIONS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Identify/suggest physical locations for the plan...") + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['identify_purpose']['markdown'].open("r") as f: + identify_purpose_markdown = f.read() + with self.input()['plan_type']['raw'].open("r") as f: + plan_type_dict = json.load(f) + with self.input()['plan_type']['markdown'].open("r") as f: + plan_type_markdown = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + + output_raw_path = self.output()['raw'].path + output_markdown_path = self.output()['markdown'].path + + plan_type = plan_type_dict.get("plan_type") + if plan_type == "physical": + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" + f"File 'plan_type.md':\n{plan_type_markdown}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}" + ) + + physical_locations = PhysicalLocations.execute(llm, query) + + # Write the physical locations to disk. + physical_locations.save_raw(str(output_raw_path)) + physical_locations.save_markdown(str(output_markdown_path)) + else: + # Write an empty file to indicate that there are no physical locations. + data = { + "comment": "The plan is purely digital, without any physical locations." + } + with open(output_raw_path, "w") as f: + json.dump(data, f, indent=2) + + with open(output_markdown_path, "w", encoding='utf-8') as f: + f.write("The plan is purely digital, without any physical locations.") diff --git a/worker_plan/worker_plan_internal/plan/stages/review_assumptions.py b/worker_plan/worker_plan_internal/plan/stages/review_assumptions.py new file mode 100644 index 000000000..17db2648d --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/review_assumptions.py @@ -0,0 +1,80 @@ +"""ReviewAssumptionsTask - Find issues with the assumptions.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.assume.review_assumptions import ReviewAssumptions +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask + +logger = logging.getLogger(__name__) + + +class ReviewAssumptionsTask(PlanTask): + """ + Find issues with the assumptions. + """ + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'currency_strategy': self.clone(CurrencyStrategyTask), + 'identify_risks': self.clone(IdentifyRisksTask), + 'make_assumptions': self.clone(MakeAssumptionsTask), + 'distill_assumptions': self.clone(DistillAssumptionsTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.REVIEW_ASSUMPTIONS_RAW), + 'markdown': self.local_target(FilenameEnum.REVIEW_ASSUMPTIONS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Define the list of (title, path) tuples + title_path_list = [ + ('Purpose', self.input()['identify_purpose']['markdown'].path), + ('Plan Type', self.input()['plan_type']['markdown'].path), + ('Strategic Decisions', self.input()['strategic_decisions_markdown']['markdown'].path), + ('Scenarios', self.input()['scenarios_markdown']['markdown'].path), + ('Physical Locations', self.input()['physical_locations']['markdown'].path), + ('Currency Strategy', self.input()['currency_strategy']['markdown'].path), + ('Identify Risks', self.input()['identify_risks']['markdown'].path), + ('Make Assumptions', self.input()['make_assumptions']['markdown'].path), + ('Distill Assumptions', self.input()['distill_assumptions']['markdown'].path) + ] + + # Read the files and handle exceptions + markdown_chunks = [] + for title, path in title_path_list: + try: + with open(path, 'r', encoding='utf-8') as f: + markdown_chunk = f.read() + markdown_chunks.append(f"# {title}\n\n{markdown_chunk}") + except FileNotFoundError: + logger.warning(f"Markdown file not found: {path} (from {title})") + markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") + except Exception as e: + logger.error(f"Error reading markdown file {path} (from {title}): {e}") + markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") + + # Combine the markdown chunks + full_markdown = "\n\n".join(markdown_chunks) + + review_assumptions = ReviewAssumptions.execute(llm, full_markdown) + + # Write the result to disk. + output_raw_path = self.output()['raw'].path + review_assumptions.save_raw(str(output_raw_path)) + output_markdown_path = self.output()['markdown'].path + review_assumptions.save_markdown(str(output_markdown_path)) From 92b915f6f1b304805d5d5c48936e9e2d7fd52c4e Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:54:16 +0200 Subject: [PATCH 05/12] refactor: extract Phase 6-7 pipeline stages (plan foundation & governance) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/consolidate_governance.py | 53 +++++++++++ .../plan/stages/governance_phase1_audit.py | 63 +++++++++++++ .../plan/stages/governance_phase2_bodies.py | 68 ++++++++++++++ .../stages/governance_phase3_impl_plan.py | 70 +++++++++++++++ ...nance_phase4_decision_escalation_matrix.py | 75 ++++++++++++++++ .../governance_phase5_monitoring_progress.py | 80 +++++++++++++++++ .../plan/stages/governance_phase6_extra.py | 90 +++++++++++++++++++ .../plan/stages/pre_project_assessment.py | 62 +++++++++++++ .../plan/stages/project_plan.py | 74 +++++++++++++++ 9 files changed, 635 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/consolidate_governance.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase1_audit.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase2_bodies.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase3_impl_plan.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase4_decision_escalation_matrix.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase5_monitoring_progress.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/governance_phase6_extra.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/pre_project_assessment.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/project_plan.py diff --git a/worker_plan/worker_plan_internal/plan/stages/consolidate_governance.py b/worker_plan/worker_plan_internal/plan/stages/consolidate_governance.py new file mode 100644 index 000000000..424ae8c40 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/consolidate_governance.py @@ -0,0 +1,53 @@ +"""ConsolidateGovernanceTask - Combines all governance phase markdown documents.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask +from worker_plan_internal.plan.stages.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgressTask +from worker_plan_internal.plan.stages.governance_phase6_extra import GovernancePhase6ExtraTask + + +class ConsolidateGovernanceTask(PlanTask): + def requires(self): + return { + 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), + 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), + 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), + 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask), + 'governance_phase6_extra': self.clone(GovernancePhase6ExtraTask) + } + + def output(self): + return self.local_target(FilenameEnum.CONSOLIDATE_GOVERNANCE_MARKDOWN) + + def run_inner(self): + # Read inputs from required tasks. + with self.input()['governance_phase1_audit']['markdown'].open("r") as f: + governance_phase1_audit_markdown = f.read() + with self.input()['governance_phase2_bodies']['markdown'].open("r") as f: + governance_phase2_bodies_markdown = f.read() + with self.input()['governance_phase3_impl_plan']['markdown'].open("r") as f: + governance_phase3_impl_plan_markdown = f.read() + with self.input()['governance_phase4_decision_escalation_matrix']['markdown'].open("r") as f: + governance_phase4_decision_escalation_matrix_markdown = f.read() + with self.input()['governance_phase5_monitoring_progress']['markdown'].open("r") as f: + governance_phase5_monitoring_progress_markdown = f.read() + with self.input()['governance_phase6_extra']['markdown'].open("r") as f: + governance_phase6_extra_markdown = f.read() + + # Build the document. + markdown = [] + markdown.append(f"# Governance Audit\n\n{governance_phase1_audit_markdown}") + markdown.append(f"# Internal Governance Bodies\n\n{governance_phase2_bodies_markdown}") + markdown.append(f"# Governance Implementation Plan\n\n{governance_phase3_impl_plan_markdown}") + markdown.append(f"# Decision Escalation Matrix\n\n{governance_phase4_decision_escalation_matrix_markdown}") + markdown.append(f"# Monitoring Progress\n\n{governance_phase5_monitoring_progress_markdown}") + markdown.append(f"# Governance Extra\n\n{governance_phase6_extra_markdown}") + + content = "\n\n".join(markdown) + + with self.output().open("w") as f: + f.write(content) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase1_audit.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase1_audit.py new file mode 100644 index 000000000..46d239c5d --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase1_audit.py @@ -0,0 +1,63 @@ +"""GovernancePhase1AuditTask - Governance audit phase.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase1_audit import GovernancePhase1Audit +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase1AuditTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE1_AUDIT_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE1_AUDIT_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}" + ) + + # Execute. + try: + governance_phase1_audit = GovernancePhase1Audit.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase1Audit failed: %s", e) + raise + + # Save the results. + governance_phase1_audit.save_raw(self.output()['raw'].path) + governance_phase1_audit.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase2_bodies.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase2_bodies.py new file mode 100644 index 000000000..708c2a246 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase2_bodies.py @@ -0,0 +1,68 @@ +"""GovernancePhase2BodiesTask - Internal governance bodies.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase2_bodies import GovernancePhase2Bodies +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase2BodiesTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE2_BODIES_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE2_BODIES_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['governance_phase1_audit']['markdown'].open("r") as f: + governance_phase1_audit_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'governance-phase1-audit.md':\n{governance_phase1_audit_markdown}" + ) + + # Execute. + try: + governance_phase2_bodies = GovernancePhase2Bodies.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase2Bodies failed: %s", e) + raise + + # Save the results. + governance_phase2_bodies.save_raw(self.output()['raw'].path) + governance_phase2_bodies.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase3_impl_plan.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase3_impl_plan.py new file mode 100644 index 000000000..55bcc12ff --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase3_impl_plan.py @@ -0,0 +1,70 @@ +"""GovernancePhase3ImplPlanTask - Governance implementation plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase3_impl_plan import GovernancePhase3ImplPlan +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase3ImplPlanTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE3_IMPL_PLAN_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE3_IMPL_PLAN_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + with self.input()['governance_phase2_bodies']['raw'].open("r") as f: + governance_phase2_bodies_dict = json.load(f) + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" + f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}" + ) + + # Execute. + try: + governance_phase3_impl_plan = GovernancePhase3ImplPlan.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase3ImplPlan failed: %s", e) + raise + + # Save the results. + governance_phase3_impl_plan.save_raw(self.output()['raw'].path) + governance_phase3_impl_plan.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase4_decision_escalation_matrix.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase4_decision_escalation_matrix.py new file mode 100644 index 000000000..cd54ba3c2 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase4_decision_escalation_matrix.py @@ -0,0 +1,75 @@ +"""GovernancePhase4DecisionEscalationMatrixTask - Decision escalation matrix.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrix +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase4DecisionEscalationMatrixTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), + 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE4_DECISION_ESCALATION_MATRIX_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE4_DECISION_ESCALATION_MATRIX_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + with self.input()['governance_phase2_bodies']['raw'].open("r") as f: + governance_phase2_bodies_dict = json.load(f) + with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: + governance_phase3_impl_plan_dict = json.load(f) + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" + f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" + f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}" + ) + + # Execute. + try: + governance_phase4_decision_escalation_matrix = GovernancePhase4DecisionEscalationMatrix.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase4DecisionEscalationMatrix failed: %s", e) + raise + + # Save the results. + governance_phase4_decision_escalation_matrix.save_raw(self.output()['raw'].path) + governance_phase4_decision_escalation_matrix.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase5_monitoring_progress.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase5_monitoring_progress.py new file mode 100644 index 000000000..16b295155 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase5_monitoring_progress.py @@ -0,0 +1,80 @@ +"""GovernancePhase5MonitoringProgressTask - Monitoring progress governance phase.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgress +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase5MonitoringProgressTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), + 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), + 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE5_MONITORING_PROGRESS_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE5_MONITORING_PROGRESS_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + with self.input()['governance_phase2_bodies']['raw'].open("r") as f: + governance_phase2_bodies_dict = json.load(f) + with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: + governance_phase3_impl_plan_dict = json.load(f) + with self.input()['governance_phase4_decision_escalation_matrix']['raw'].open("r") as f: + governance_phase4_decision_escalation_matrix_dict = json.load(f) + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" + f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" + f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}\n\n" + f"File 'governance-phase4-decision-escalation-matrix.json':\n{format_json_for_use_in_query(governance_phase4_decision_escalation_matrix_dict)}" + ) + + # Execute. + try: + governance_phase5_monitoring_progress = GovernancePhase5MonitoringProgress.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase5MonitoringProgress failed: %s", e) + raise + + # Save the results. + governance_phase5_monitoring_progress.save_raw(self.output()['raw'].path) + governance_phase5_monitoring_progress.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/governance_phase6_extra.py b/worker_plan/worker_plan_internal/plan/stages/governance_phase6_extra.py new file mode 100644 index 000000000..e772622d3 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/governance_phase6_extra.py @@ -0,0 +1,90 @@ +"""GovernancePhase6ExtraTask - Extra governance considerations.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.governance.governance_phase6_extra import GovernancePhase6Extra +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask +from worker_plan_internal.plan.stages.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgressTask + +logger = logging.getLogger(__name__) + + +class GovernancePhase6ExtraTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), + 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), + 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), + 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE6_EXTRA_RAW), + 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE6_EXTRA_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + with self.input()['governance_phase1_audit']['raw'].open("r") as f: + governance_phase1_audit_dict = json.load(f) + with self.input()['governance_phase2_bodies']['raw'].open("r") as f: + governance_phase2_bodies_dict = json.load(f) + with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: + governance_phase3_impl_plan_dict = json.load(f) + with self.input()['governance_phase4_decision_escalation_matrix']['raw'].open("r") as f: + governance_phase4_decision_escalation_matrix_dict = json.load(f) + with self.input()['governance_phase5_monitoring_progress']['raw'].open("r") as f: + governance_phase5_monitoring_progress_dict = json.load(f) + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" + f"File 'governance-phase1-audit.json':\n{format_json_for_use_in_query(governance_phase1_audit_dict)}\n\n" + f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" + f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}\n\n" + f"File 'governance-phase4-decision-escalation-matrix.json':\n{format_json_for_use_in_query(governance_phase4_decision_escalation_matrix_dict)}\n\n" + f"File 'governance-phase5-monitoring-progress.json':\n{format_json_for_use_in_query(governance_phase5_monitoring_progress_dict)}" + ) + + # Execute. + try: + governance_phase6_extra = GovernancePhase6Extra.execute(llm, query) + except Exception as e: + logger.error("GovernancePhase6Extra failed: %s", e) + raise + + # Save the results. + governance_phase6_extra.save_raw(self.output()['raw'].path) + governance_phase6_extra.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/pre_project_assessment.py b/worker_plan/worker_plan_internal/plan/stages/pre_project_assessment.py new file mode 100644 index 000000000..a026861cc --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/pre_project_assessment.py @@ -0,0 +1,62 @@ +"""PreProjectAssessmentTask - Conducts pre-project assessment.""" +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.expert.pre_project_assessment import PreProjectAssessment +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask + +logger = logging.getLogger(__name__) + + +class PreProjectAssessmentTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PRE_PROJECT_ASSESSMENT_RAW), + 'clean': self.local_target(FilenameEnum.PRE_PROJECT_ASSESSMENT) + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Conducting pre-project assessment...") + + # Read the plan prompt from the SetupTask's output. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + + # Build the query. + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}" + ) + + # Execute the pre-project assessment. + pre_project_assessment = PreProjectAssessment.execute(llm, query) + + # Save raw output. + raw_path = self.file_path(FilenameEnum.PRE_PROJECT_ASSESSMENT_RAW) + pre_project_assessment.save_raw(str(raw_path)) + + # Save cleaned pre-project assessment. + clean_path = self.file_path(FilenameEnum.PRE_PROJECT_ASSESSMENT) + pre_project_assessment.save_preproject_assessment(str(clean_path)) diff --git a/worker_plan/worker_plan_internal/plan/stages/project_plan.py b/worker_plan/worker_plan_internal/plan/stages/project_plan.py new file mode 100644 index 000000000..f63fa3c80 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/project_plan.py @@ -0,0 +1,74 @@ +"""ProjectPlanTask - Creates the project plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.project_plan import ProjectPlan +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask + +logger = logging.getLogger(__name__) + + +class ProjectPlanTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PROJECT_PLAN_RAW), + 'markdown': self.local_target(FilenameEnum.PROJECT_PLAN_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Creating plan...") + + # Read the plan prompt from SetupTask's output. + setup_target = self.input()['setup'] + with setup_target.open("r") as f: + plan_prompt = f.read() + + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + + # Load the consolidated assumptions. + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + + # Read the pre-project assessment from its file. + pre_project_assessment_file = self.input()['preproject']['clean'] + with pre_project_assessment_file.open("r") as f: + pre_project_assessment_dict = json.load(f) + + # Build the query. + query = ( + f"File 'plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}" + ) + + # Execute the plan creation. + project_plan = ProjectPlan.execute(llm, query) + + # Save raw output + project_plan.save_raw(self.output()['raw'].path) + + # Save markdown output + project_plan.save_markdown(self.output()['markdown'].path) + + logger.info("Project plan created and saved") From f5a72a1203431d8ded897472e9f7a2fcd216b8a5 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:56:36 +0200 Subject: [PATCH 06/12] refactor: extract Phase 8 pipeline stages (team) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../stages/enrich_team_background_story.py | 99 +++++++++++++++++++ .../plan/stages/enrich_team_contract_type.py | 86 ++++++++++++++++ .../stages/enrich_team_environment_info.py | 86 ++++++++++++++++ .../plan/stages/find_team_members.py | 81 +++++++++++++++ .../plan/stages/related_resources.py | 65 ++++++++++++ .../plan/stages/review_team.py | 86 ++++++++++++++++ .../plan/stages/team_markdown.py | 44 +++++++++ 7 files changed, 547 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/enrich_team_background_story.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/enrich_team_contract_type.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/enrich_team_environment_info.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/find_team_members.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/related_resources.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/review_team.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/team_markdown.py diff --git a/worker_plan/worker_plan_internal/plan/stages/enrich_team_background_story.py b/worker_plan/worker_plan_internal/plan/stages/enrich_team_background_story.py new file mode 100644 index 000000000..7ebcb8481 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/enrich_team_background_story.py @@ -0,0 +1,99 @@ +"""EnrichTeamMembersWithBackgroundStoryTask - Enriches team members with background stories.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.enrich_team_members_with_background_story import EnrichTeamMembersWithBackgroundStory +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.enrich_team_contract_type import EnrichTeamMembersWithContractTypeTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class EnrichTeamMembersWithBackgroundStoryTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'enrich_team_members_with_contract_type': self.clone(EnrichTeamMembersWithContractTypeTask), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_BACKGROUND_STORY_RAW), + 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_BACKGROUND_STORY_CLEAN) + } + + def run_with_llm(self, llm: LLM) -> None: + # In FAST_BUT_SKIP_DETAILS mode, skip fictional biography generation entirely. + # Pass the bare team member list (title, domain, relevance) through unchanged. + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode: skipping biography enrichment.") + with self.input()['enrich_team_members_with_contract_type']['clean'].open("r") as f: + team_member_list = json.load(f) + with self.output()['raw'].open("w") as f: + json.dump({"team_members": team_member_list}, f, indent=2) + with self.output()['clean'].open("w") as f: + json.dump(team_member_list, f, indent=2) + return + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['enrich_team_members_with_contract_type']['clean'].open("r") as f: + team_member_list = json.load(f) + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute. + try: + enrich_team_members_with_background_story = EnrichTeamMembersWithBackgroundStory.execute(llm, query, team_member_list) + except Exception as e: + logger.error("EnrichTeamMembersWithBackgroundStory failed: %s", e) + raise + + # Save the raw output. + raw_dict = enrich_team_members_with_background_story.to_dict() + with self.output()['raw'].open("w") as f: + json.dump(raw_dict, f, indent=2) + + # Save the cleaned up result. + team_member_list = enrich_team_members_with_background_story.team_member_list + with self.output()['clean'].open("w") as f: + json.dump(team_member_list, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/enrich_team_contract_type.py b/worker_plan/worker_plan_internal/plan/stages/enrich_team_contract_type.py new file mode 100644 index 000000000..a6b522928 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/enrich_team_contract_type.py @@ -0,0 +1,86 @@ +"""EnrichTeamMembersWithContractTypeTask - Enriches team members with contract type information.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.enrich_team_members_with_contract_type import EnrichTeamMembersWithContractType +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.find_team_members import FindTeamMembersTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class EnrichTeamMembersWithContractTypeTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'find_team_members': self.clone(FindTeamMembersTask), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_CONTRACT_TYPE_RAW), + 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_CONTRACT_TYPE_CLEAN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['find_team_members']['clean'].open("r") as f: + team_member_list = json.load(f) + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute. + try: + enrich_team_members_with_contract_type = EnrichTeamMembersWithContractType.execute(llm, query, team_member_list) + except Exception as e: + logger.error("EnrichTeamMembersWithContractType failed: %s", e) + raise + + # Save the raw output. + raw_dict = enrich_team_members_with_contract_type.to_dict() + with self.output()['raw'].open("w") as f: + json.dump(raw_dict, f, indent=2) + + # Save the cleaned up result. + team_member_list = enrich_team_members_with_contract_type.team_member_list + with self.output()['clean'].open("w") as f: + json.dump(team_member_list, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/enrich_team_environment_info.py b/worker_plan/worker_plan_internal/plan/stages/enrich_team_environment_info.py new file mode 100644 index 000000000..6791fe517 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/enrich_team_environment_info.py @@ -0,0 +1,86 @@ +"""EnrichTeamMembersWithEnvironmentInfoTask - Enriches team members with environment information.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.enrich_team_members_with_environment_info import EnrichTeamMembersWithEnvironmentInfo +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.enrich_team_background_story import EnrichTeamMembersWithBackgroundStoryTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class EnrichTeamMembersWithEnvironmentInfoTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'enrich_team_members_with_background_story': self.clone(EnrichTeamMembersWithBackgroundStoryTask), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_ENVIRONMENT_INFO_RAW), + 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_ENVIRONMENT_INFO_CLEAN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['enrich_team_members_with_background_story']['clean'].open("r") as f: + team_member_list = json.load(f) + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute. + try: + enrich_team_members_with_background_story = EnrichTeamMembersWithEnvironmentInfo.execute(llm, query, team_member_list) + except Exception as e: + logger.error("EnrichTeamMembersWithEnvironmentInfo failed: %s", e) + raise + + # Save the raw output. + raw_dict = enrich_team_members_with_background_story.to_dict() + with self.output()['raw'].open("w") as f: + json.dump(raw_dict, f, indent=2) + + # Save the cleaned up result. + team_member_list = enrich_team_members_with_background_story.team_member_list + with self.output()['clean'].open("w") as f: + json.dump(team_member_list, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/find_team_members.py b/worker_plan/worker_plan_internal/plan/stages/find_team_members.py new file mode 100644 index 000000000..c5cefb609 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/find_team_members.py @@ -0,0 +1,81 @@ +"""FindTeamMembersTask - Identifies team members needed for the project.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.find_team_members import FindTeamMembers +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class FindTeamMembersTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'related_resources': self.clone(RelatedResourcesTask), + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.FIND_TEAM_MEMBERS_RAW), + 'clean': self.local_target(FilenameEnum.FIND_TEAM_MEMBERS_CLEAN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute. + try: + find_team_members = FindTeamMembers.execute(llm, query) + except Exception as e: + logger.error("FindTeamMembers failed: %s", e) + raise + + # Save the raw output. + raw_dict = find_team_members.to_dict() + with self.output()['raw'].open("w") as f: + json.dump(raw_dict, f, indent=2) + + # Save the cleaned up result. + team_member_list = find_team_members.team_member_list + with self.output()['clean'].open("w") as f: + json.dump(team_member_list, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/related_resources.py b/worker_plan/worker_plan_internal/plan/stages/related_resources.py new file mode 100644 index 000000000..66a6cad8a --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/related_resources.py @@ -0,0 +1,65 @@ +"""RelatedResourcesTask - Finds related resources for the project.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.related_resources import RelatedResources +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask + +logger = logging.getLogger(__name__) + + +class RelatedResourcesTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.RELATED_RESOURCES_RAW), + 'markdown': self.local_target(FilenameEnum.RELATED_RESOURCES_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}" + ) + + # Execute. + try: + related_resources = RelatedResources.execute(llm, query) + except Exception as e: + logger.error("SimilarProjects failed: %s", e) + raise + + # Save the results. + related_resources.save_raw(self.output()['raw'].path) + related_resources.save_markdown(self.output()['markdown'].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/review_team.py b/worker_plan/worker_plan_internal/plan/stages/review_team.py new file mode 100644 index 000000000..2567d315a --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/review_team.py @@ -0,0 +1,86 @@ +"""ReviewTeamTask - Reviews the assembled team.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.review_team import ReviewTeam +from worker_plan_internal.team.team_markdown_document import TeamMarkdownDocumentBuilder +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.enrich_team_environment_info import EnrichTeamMembersWithEnvironmentInfoTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class ReviewTeamTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def output(self): + return self.local_target(FilenameEnum.REVIEW_TEAM_RAW) + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['enrich_team_members_with_environment_info']['clean'].open("r") as f: + team_member_list = json.load(f) + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Convert the team members to a Markdown document. + builder = TeamMarkdownDocumentBuilder() + builder.append_roles(team_member_list, title=None) + team_document_markdown = builder.to_string() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'team-members.md':\n{team_document_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute. + try: + review_team = ReviewTeam.execute(llm, query) + except Exception as e: + logger.error("ReviewTeam failed: %s", e) + raise + + # Save the raw output. + raw_dict = review_team.to_dict() + with self.output().open("w") as f: + json.dump(raw_dict, f, indent=2) + + logger.info("ReviewTeamTask complete.") diff --git a/worker_plan/worker_plan_internal/plan/stages/team_markdown.py b/worker_plan/worker_plan_internal/plan/stages/team_markdown.py new file mode 100644 index 000000000..ffd7150a0 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/team_markdown.py @@ -0,0 +1,44 @@ +"""TeamMarkdownTask - Generates the final team Markdown document.""" +import json +import logging +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.team.team_markdown_document import TeamMarkdownDocumentBuilder +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.enrich_team_environment_info import EnrichTeamMembersWithEnvironmentInfoTask +from worker_plan_internal.plan.stages.review_team import ReviewTeamTask + +logger = logging.getLogger(__name__) + + +class TeamMarkdownTask(PlanTask): + def requires(self): + return { + 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), + 'review_team': self.clone(ReviewTeamTask) + } + + def output(self): + return self.local_target(FilenameEnum.TEAM_MARKDOWN) + + def run_inner(self): + logger.info("TeamMarkdownTask. Loading files...") + + # 1. Read the team_member_list from EnrichTeamMembersWithEnvironmentInfoTask. + with self.input()['enrich_team_members_with_environment_info']['clean'].open("r") as f: + team_member_list = json.load(f) + + # 2. Read the json from ReviewTeamTask. + with self.input()['review_team'].open("r") as f: + review_team_json = json.load(f) + + logger.info("TeamMarkdownTask. All files are now ready. Processing...") + + # Combine the team members and the review into a Markdown document. + builder = TeamMarkdownDocumentBuilder() + builder.append_team_member_subtitle() + builder.append_roles(team_member_list) + builder.append_separator() + builder.append_full_review(review_team_json) + builder.write_to_file(self.output().path) + + logger.info("TeamMarkdownTask complete.") From 4aaff8c7931434c64556a7679870b5cf11a297cc Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 18:59:45 +0200 Subject: [PATCH 07/12] refactor: extract Phase 9-10 pipeline stages (analysis & documents) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/data_collection.py | 76 +++++++++++++ .../plan/stages/draft_documents_to_create.py | 104 ++++++++++++++++++ .../plan/stages/draft_documents_to_find.py | 104 ++++++++++++++++++ .../plan/stages/expert_review.py | 90 +++++++++++++++ .../plan/stages/filter_documents_to_create.py | 72 ++++++++++++ .../plan/stages/filter_documents_to_find.py | 72 ++++++++++++ .../plan/stages/identify_documents.py | 87 +++++++++++++++ .../plan/stages/markdown_documents.py | 46 ++++++++ .../plan/stages/swot_analysis.py | 89 +++++++++++++++ 9 files changed, 740 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/data_collection.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/draft_documents_to_create.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/draft_documents_to_find.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/expert_review.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/filter_documents_to_create.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/filter_documents_to_find.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/identify_documents.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/markdown_documents.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/swot_analysis.py diff --git a/worker_plan/worker_plan_internal/plan/stages/data_collection.py b/worker_plan/worker_plan_internal/plan/stages/data_collection.py new file mode 100644 index 000000000..86de18c4c --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/data_collection.py @@ -0,0 +1,76 @@ +"""DataCollectionTask - Determines what kind of data is to be collected.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.data_collection import DataCollection +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask + + +class DataCollectionTask(PlanTask): + """ + Determine what kind of data is to be collected. + """ + def output(self): + return { + 'raw': self.local_target(FilenameEnum.DATA_COLLECTION_RAW), + 'markdown': self.local_target(FilenameEnum.DATA_COLLECTION_MARKDOWN) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'expert_review': self.clone(ExpertReviewTask) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}" + ) + + # Invoke the LLM. + data_collection = DataCollection.execute(llm, query) + + # Save the results. + json_path = self.output()['raw'].path + data_collection.save_raw(json_path) + markdown_path = self.output()['markdown'].path + data_collection.save_markdown(markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_create.py b/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_create.py new file mode 100644 index 000000000..47a6e7d2b --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_create.py @@ -0,0 +1,104 @@ +"""DraftDocumentsToCreateTask - Drafts bullet points for documents to create.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.draft_document_to_create import DraftDocumentToCreate +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_api.filenames import FilenameEnum +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.filter_documents_to_create import FilterDocumentsToCreateTask + +logger = logging.getLogger(__name__) + + +class DraftDocumentsToCreateTask(PlanTask): + """ + The "documents to create". Write bullet points to what each document roughly should contain. + """ + def output(self): + return self.local_target(FilenameEnum.DRAFT_DOCUMENTS_TO_CREATE_CONSOLIDATED) + + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'filter_documents_to_create': self.clone(FilterDocumentsToCreateTask), + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['filter_documents_to_create']['clean'].open("r") as f: + documents_to_create = json.load(f) + + accumulated_documents = documents_to_create.copy() + + logger.info(f"DraftDocumentsToCreateTask.speedvsdetail: {self.speedvsdetail}") + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") + documents_to_create = documents_to_create[:2] + else: + logger.info("Processing all chunks.") + + for index, document in enumerate(documents_to_create): + logger.info(f"Document-to-create: Drafting document {index+1} of {len(documents_to_create)}...") + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'document.json':\n{document}" + ) + + # IDEA: If the document already exist, then there is no need to run the LLM again. + def execute_draft_document_to_create(llm: LLM) -> DraftDocumentToCreate: + return DraftDocumentToCreate.execute(llm=llm, user_prompt=query, identify_purpose_dict=identify_purpose_dict) + + try: + draft_document = llm_executor.run(execute_draft_document_to_create) + except PipelineStopRequested: + # Re-raise PipelineStopRequested without wrapping it + raise + except Exception as e: + logger.error(f"Document-to-create {index+1} LLM interaction failed.", exc_info=True) + raise ValueError(f"Document-to-create {index+1} LLM interaction failed.") from e + + json_response = draft_document.to_dict() + + # Write the raw JSON for this document using the FilenameEnum template. + raw_filename = FilenameEnum.DRAFT_DOCUMENTS_TO_CREATE_RAW_TEMPLATE.value.format(index+1) + raw_chunk_path = self.run_id_dir / raw_filename + with open(raw_chunk_path, 'w') as f: + json.dump(json_response, f, indent=2) + + # Merge the draft document into the original document. + document_updated = document.copy() + for key in draft_document.response.keys(): + document_updated[key] = draft_document.response[key] + accumulated_documents[index] = document_updated + + # Write the accumulated documents to the output file. + with self.output().open("w") as f: + json.dump(accumulated_documents, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_find.py b/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_find.py new file mode 100644 index 000000000..658429989 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/draft_documents_to_find.py @@ -0,0 +1,104 @@ +"""DraftDocumentsToFindTask - Drafts bullet points for documents to find.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.draft_document_to_find import DraftDocumentToFind +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_api.filenames import FilenameEnum +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.filter_documents_to_find import FilterDocumentsToFindTask + +logger = logging.getLogger(__name__) + + +class DraftDocumentsToFindTask(PlanTask): + """ + The "documents to find". Write bullet points to what each document roughly should contain. + """ + def output(self): + return self.local_target(FilenameEnum.DRAFT_DOCUMENTS_TO_FIND_CONSOLIDATED) + + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'filter_documents_to_find': self.clone(FilterDocumentsToFindTask), + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['filter_documents_to_find']['clean'].open("r") as f: + documents_to_find = json.load(f) + + accumulated_documents = documents_to_find.copy() + + logger.info(f"DraftDocumentsToFindTask.speedvsdetail: {self.speedvsdetail}") + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") + documents_to_find = documents_to_find[:2] + else: + logger.info("Processing all chunks.") + + for index, document in enumerate(documents_to_find): + logger.info(f"Document-to-find: Drafting document {index+1} of {len(documents_to_find)}...") + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'document.json':\n{document}" + ) + + # IDEA: If the document already exist, then there is no need to run the LLM again. + def execute_draft_document_to_find(llm: LLM) -> DraftDocumentToFind: + return DraftDocumentToFind.execute(llm=llm, user_prompt=query, identify_purpose_dict=identify_purpose_dict) + + try: + draft_document = llm_executor.run(execute_draft_document_to_find) + except PipelineStopRequested: + # Re-raise PipelineStopRequested without wrapping it + raise + except Exception as e: + logger.error(f"Document-to-find {index+1} LLM interaction failed.", exc_info=True) + raise ValueError(f"Document-to-find {index+1} LLM interaction failed.") from e + + json_response = draft_document.to_dict() + + # Write the raw JSON for this document using the FilenameEnum template. + raw_filename = FilenameEnum.DRAFT_DOCUMENTS_TO_FIND_RAW_TEMPLATE.value.format(index+1) + raw_chunk_path = self.run_id_dir / raw_filename + with open(raw_chunk_path, 'w') as f: + json.dump(json_response, f, indent=2) + + # Merge the draft document into the original document. + document_updated = document.copy() + for key in draft_document.response.keys(): + document_updated[key] = draft_document.response[key] + accumulated_documents[index] = document_updated + + # Write the accumulated documents to the output file. + with self.output().open("w") as f: + json.dump(accumulated_documents, f, indent=2) diff --git a/worker_plan/worker_plan_internal/plan/stages/expert_review.py b/worker_plan/worker_plan_internal/plan/stages/expert_review.py new file mode 100644 index 000000000..05b8635fc --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/expert_review.py @@ -0,0 +1,90 @@ +"""ExpertReviewTask - Finds experts to review the SWOT analysis and have them provide criticism.""" +import json +import logging +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.expert.expert_finder import ExpertFinder +from worker_plan_internal.expert.expert_criticism import ExpertCriticism +from worker_plan_internal.expert.expert_orchestrator import ExpertOrchestrator +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask + +logger = logging.getLogger(__name__) + + +class ExpertReviewTask(PlanTask): + """ + Finds experts to review the SWOT analysis and have them provide criticism. + """ + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'swot_analysis': self.clone(SWOTAnalysisTask) + } + + def output(self): + return self.local_target(FilenameEnum.EXPERT_CRITICISM_MARKDOWN) + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + logger.info("Finding experts to review the SWOT analysis, and having them provide criticism...") + + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + swot_markdown_path = self.input()['swot_analysis']['markdown'].path + with open(swot_markdown_path, "r", encoding="utf-8") as f: + swot_markdown = f.read() + + # Build the query. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'pre-project assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project_plan.md':\n{project_plan_markdown}\n\n" + f"File 'SWOT Analysis.md':\n{swot_markdown}" + ) + + # Define callback functions. + def phase1_post_callback(expert_finder: ExpertFinder) -> None: + raw_path = self.run_id_dir / FilenameEnum.EXPERTS_RAW.value + cleaned_path = self.run_id_dir / FilenameEnum.EXPERTS_CLEAN.value + expert_finder.save_raw(str(raw_path)) + expert_finder.save_cleanedup(str(cleaned_path)) + + def phase2_post_callback(expert_criticism: ExpertCriticism, expert_index: int) -> None: + file_path = self.run_id_dir / FilenameEnum.EXPERT_CRITICISM_RAW_TEMPLATE.format(expert_index + 1) + expert_criticism.save_raw(str(file_path)) + + # Execute the expert orchestration. + expert_orchestrator = ExpertOrchestrator() + # IDEA: max_expert_count. don't truncate to 2 experts. Interview them all in production mode. + # IDEA: If the expert file for expert_index already exist, then there is no need to run the LLM again. + expert_orchestrator.phase1_post_callback = phase1_post_callback + expert_orchestrator.phase2_post_callback = phase2_post_callback + expert_orchestrator.execute(llm_executor, query) + + # Write final expert criticism markdown. + expert_criticism_markdown_file = self.file_path(FilenameEnum.EXPERT_CRITICISM_MARKDOWN) + with expert_criticism_markdown_file.open("w") as f: + f.write(expert_orchestrator.to_markdown()) diff --git a/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_create.py b/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_create.py new file mode 100644 index 000000000..179b4c253 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_create.py @@ -0,0 +1,72 @@ +"""FilterDocumentsToCreateTask - Narrows down documents to create to a relevant subset.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.filter_documents_to_create import FilterDocumentsToCreate +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.identify_documents import IdentifyDocumentsTask + + +class FilterDocumentsToCreateTask(PlanTask): + """ + The "documents to create" may be a long list of documents, some duplicates, irrelevant, not needed at an early stage of the project. + This task narrows down to a handful of relevant documents. + """ + def output(self): + return { + "raw": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_CREATE_RAW), + "clean": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_CREATE_CLEAN) + } + + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'identified_documents': self.clone(IdentifyDocumentsTask), + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['identified_documents']['documents_to_create'].open("r") as f: + documents_to_create = json.load(f) + + # Build the query. + process_documents, integer_id_to_document_uuid = FilterDocumentsToCreate.process_documents_and_integer_ids(documents_to_create) + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'documents.json':\n{process_documents}" + ) + + # Invoke the LLM. + filter_documents = FilterDocumentsToCreate.execute( + llm=llm, + user_prompt=query, + identified_documents_raw_json=documents_to_create, + integer_id_to_document_uuid=integer_id_to_document_uuid, + identify_purpose_dict=identify_purpose_dict + ) + + # Save the results. + filter_documents.save_raw(self.output()["raw"].path) + filter_documents.save_filtered_documents(self.output()["clean"].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_find.py b/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_find.py new file mode 100644 index 000000000..6b7f81ab7 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/filter_documents_to_find.py @@ -0,0 +1,72 @@ +"""FilterDocumentsToFindTask - Narrows down documents to find to a relevant subset.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.filter_documents_to_find import FilterDocumentsToFind +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.identify_documents import IdentifyDocumentsTask + + +class FilterDocumentsToFindTask(PlanTask): + """ + The "documents to find" may be a long list of documents, some duplicates, irrelevant, not needed at an early stage of the project. + This task narrows down to a handful of relevant documents. + """ + def output(self): + return { + "raw": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_FIND_RAW), + "clean": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_FIND_CLEAN) + } + + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'identified_documents': self.clone(IdentifyDocumentsTask), + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['identified_documents']['documents_to_find'].open("r") as f: + documents_to_find = json.load(f) + + # Build the query. + process_documents, integer_id_to_document_uuid = FilterDocumentsToFind.process_documents_and_integer_ids(documents_to_find) + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'documents.json':\n{process_documents}" + ) + + # Invoke the LLM. + filter_documents = FilterDocumentsToFind.execute( + llm=llm, + user_prompt=query, + identified_documents_raw_json=documents_to_find, + integer_id_to_document_uuid=integer_id_to_document_uuid, + identify_purpose_dict=identify_purpose_dict + ) + + # Save the results. + filter_documents.save_raw(self.output()["raw"].path) + filter_documents.save_filtered_documents(self.output()["clean"].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/identify_documents.py b/worker_plan/worker_plan_internal/plan/stages/identify_documents.py new file mode 100644 index 000000000..b09706966 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/identify_documents.py @@ -0,0 +1,87 @@ +"""IdentifyDocumentsTask - Identifies documents that need to be created or found.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.identify_documents import IdentifyDocuments +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask + + +class IdentifyDocumentsTask(PlanTask): + """ + Identify documents that need to be created or found for the project. + """ + def output(self): + return { + "raw": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_RAW), + "markdown": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_MARKDOWN), + "documents_to_find": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_TO_FIND_JSON), + "documents_to_create": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_TO_CREATE_JSON), + } + + def requires(self): + return { + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'expert_review': self.clone(ExpertReviewTask) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}" + ) + + # Invoke the LLM. + identify_documents = IdentifyDocuments.execute( + llm=llm, + user_prompt=query, + identify_purpose_dict=identify_purpose_dict + ) + + # Save the results. + identify_documents.save_raw(self.output()["raw"].path) + identify_documents.save_markdown(self.output()["markdown"].path) + identify_documents.save_json_documents_to_find(self.output()["documents_to_find"].path) + identify_documents.save_json_documents_to_create(self.output()["documents_to_create"].path) diff --git a/worker_plan/worker_plan_internal/plan/stages/markdown_documents.py b/worker_plan/worker_plan_internal/plan/stages/markdown_documents.py new file mode 100644 index 000000000..066c84ac1 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/markdown_documents.py @@ -0,0 +1,46 @@ +"""MarkdownWithDocumentsToCreateAndFindTask - Creates markdown with documents to create and find.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.document.markdown_with_document import markdown_rows_with_document_to_create, markdown_rows_with_document_to_find +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.draft_documents_to_create import DraftDocumentsToCreateTask +from worker_plan_internal.plan.stages.draft_documents_to_find import DraftDocumentsToFindTask + + +class MarkdownWithDocumentsToCreateAndFindTask(PlanTask): + """ + Create markdown with the "documents to create and find" + """ + def output(self): + return self.local_target(FilenameEnum.DOCUMENTS_TO_CREATE_AND_FIND_MARKDOWN) + + def requires(self): + return { + 'draft_documents_to_create': self.clone(DraftDocumentsToCreateTask), + 'draft_documents_to_find': self.clone(DraftDocumentsToFindTask), + } + + def run_inner(self): + # Read inputs from required tasks. + with self.input()['draft_documents_to_create'].open("r") as f: + documents_to_create = json.load(f) + with self.input()['draft_documents_to_find'].open("r") as f: + documents_to_find = json.load(f) + + accumulated_rows = [] + accumulated_rows.append("# Documents to Create") + for index, document in enumerate(documents_to_create, start=1): + rows = markdown_rows_with_document_to_create(index, document) + accumulated_rows.extend(rows) + + accumulated_rows.append("\n\n# Documents to Find") + for index, document in enumerate(documents_to_find, start=1): + rows = markdown_rows_with_document_to_find(index, document) + accumulated_rows.extend(rows) + + markdown_representation = "\n".join(accumulated_rows) + + # Write the markdown to the output file. + output_file_path = self.output().path + with open(output_file_path, 'w', encoding='utf-8') as f: + f.write(markdown_representation) diff --git a/worker_plan/worker_plan_internal/plan/stages/swot_analysis.py b/worker_plan/worker_plan_internal/plan/stages/swot_analysis.py new file mode 100644 index 000000000..34bb2bb5e --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/swot_analysis.py @@ -0,0 +1,89 @@ +"""SWOTAnalysisTask - Performs SWOT analysis on the project plan.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.swot.swot_analysis import SWOTAnalysis +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class SWOTAnalysisTask(PlanTask): + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'preproject': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.SWOT_RAW), + 'markdown': self.local_target(FilenameEnum.SWOT_MARKDOWN) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['setup'].open("r") as f: + plan_prompt = f.read() + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['identify_purpose']['raw'].open("r") as f: + identify_purpose_dict = json.load(f) + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + consolidate_assumptions_markdown = f.read() + with self.input()['preproject']['clean'].open("r") as f: + pre_project_assessment_dict = json.load(f) + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query for SWOT analysis. + query = ( + f"File 'initial-plan.txt':\n{plan_prompt}\n\n" + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" + f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}" + ) + + # Execute the SWOT analysis. + # Send the identify_purpose_dict to SWOTAnalysis, and use business/personal/other to select the system prompt + try: + swot_analysis = SWOTAnalysis.execute(llm=llm, query=query, identify_purpose_dict=identify_purpose_dict) + except Exception as e: + logger.error("SWOT analysis failed: %s", e) + raise + + # Convert the SWOT analysis to a dict and markdown. + swot_raw_dict = swot_analysis.to_dict() + swot_markdown = swot_analysis.to_markdown(include_metadata=False, include_purpose=False) + + # Write the raw SWOT JSON. + with self.output()['raw'].open("w") as f: + json.dump(swot_raw_dict, f, indent=2) + + # Write the SWOT analysis as Markdown. + markdown_path = self.output()['markdown'].path + with open(markdown_path, "w", encoding="utf-8") as f: + f.write(swot_markdown) From 62d287bc010594c02c2a2321976caa21b46411d6 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 19:02:50 +0200 Subject: [PATCH 08/12] refactor: extract Phase 11 pipeline stages (WBS & pitch) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/convert_pitch_to_markdown.py | 44 ++++++ .../plan/stages/create_pitch.py | 75 ++++++++++ .../plan/stages/create_wbs_level1.py | 62 ++++++++ .../plan/stages/create_wbs_level2.py | 78 ++++++++++ .../plan/stages/create_wbs_level3.py | 136 ++++++++++++++++++ .../plan/stages/estimate_task_durations.py | 117 +++++++++++++++ .../plan/stages/identify_task_dependencies.py | 66 +++++++++ .../stages/wbs_project_level1_and_level2.py | 35 +++++ .../wbs_project_level1_level2_level3.py | 46 ++++++ 9 files changed, 659 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/convert_pitch_to_markdown.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/create_pitch.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/create_wbs_level1.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/create_wbs_level2.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/create_wbs_level3.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/estimate_task_durations.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/identify_task_dependencies.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_and_level2.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_level2_level3.py diff --git a/worker_plan/worker_plan_internal/plan/stages/convert_pitch_to_markdown.py b/worker_plan/worker_plan_internal/plan/stages/convert_pitch_to_markdown.py new file mode 100644 index 000000000..438560063 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/convert_pitch_to_markdown.py @@ -0,0 +1,44 @@ +"""ConvertPitchToMarkdownTask - Human readable version of the pitch.""" +import json +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.pitch.convert_pitch_to_markdown import ConvertPitchToMarkdown +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.create_pitch import CreatePitchTask + + +class ConvertPitchToMarkdownTask(PlanTask): + """ + Human readable version of the pitch. + + This task depends on: + - CreatePitchTask: Creates the pitch JSON. + """ + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PITCH_CONVERT_TO_MARKDOWN_RAW), + 'markdown': self.local_target(FilenameEnum.PITCH_MARKDOWN) + } + + def requires(self): + return { + 'pitch': self.clone(CreatePitchTask), + } + + def run_with_llm(self, llm: LLM) -> None: + # Read the project plan JSON. + with self.input()['pitch'].open("r") as f: + pitch_json = json.load(f) + + # Build the query + query = format_json_for_use_in_query(pitch_json) + + # Execute the conversion. + converted = ConvertPitchToMarkdown.execute(llm, query) + + # Save the results. + json_path = self.output()['raw'].path + converted.save_raw(json_path) + markdown_path = self.output()['markdown'].path + converted.save_markdown(markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/create_pitch.py b/worker_plan/worker_plan_internal/plan/stages/create_pitch.py new file mode 100644 index 000000000..6a7594300 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/create_pitch.py @@ -0,0 +1,75 @@ +"""CreatePitchTask - Create a pitch that explains the project plan from multiple perspectives.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.pitch.create_pitch import CreatePitch +from worker_plan_internal.wbs.wbs_task import WBSProject +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask + +logger = logging.getLogger(__name__) + + +class CreatePitchTask(PlanTask): + """ + Create a the pitch that explains the project plan, from multiple perspectives. + + This task depends on: + - ProjectPlanTask: provides the project plan JSON. + - WBSProjectLevel1AndLevel2Task: containing the top level of the project plan. + + The resulting pitch JSON is written to the file specified by FilenameEnum.PITCH. + """ + def output(self): + return self.local_target(FilenameEnum.PITCH_RAW) + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), + 'related_resources': self.clone(RelatedResourcesTask) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read the project plan JSON. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + + with self.input()['wbs_project'].open("r") as f: + wbs_project_dict = json.load(f) + wbs_project = WBSProject.from_dict(wbs_project_dict) + wbs_project_json = wbs_project.to_dict() + + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + + # Build the query + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'project_plan.md':\n{project_plan_markdown}\n\n" + f"File 'Work Breakdown Structure.json':\n{format_json_for_use_in_query(wbs_project_json)}\n\n" + f"File 'similar_projects.md':\n{related_resources_markdown}" + ) + + # Execute the pitch creation. + create_pitch = CreatePitch.execute(llm, query) + pitch_dict = create_pitch.raw_response_dict() + + # Write the resulting pitch JSON to the output file. + with self.output().open("w") as f: + json.dump(pitch_dict, f, indent=2) + + logger.info("Pitch created and written to %s", self.output().path) diff --git a/worker_plan/worker_plan_internal/plan/stages/create_wbs_level1.py b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level1.py new file mode 100644 index 000000000..5755d8ed3 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level1.py @@ -0,0 +1,62 @@ +"""CreateWBSLevel1Task - Creates the Work Breakdown Structure (WBS) Level 1.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.create_wbs_level1 import CreateWBSLevel1 +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask + +logger = logging.getLogger(__name__) + + +class CreateWBSLevel1Task(PlanTask): + """ + Creates the Work Breakdown Structure (WBS) Level 1. + Depends on: + - ProjectPlanTask: provides the project plan as JSON. + Produces: + - Raw WBS Level 1 output file (xxx-wbs_level1_raw.json) + - Cleaned up WBS Level 1 file (xxx-wbs_level1.json) + """ + def requires(self): + return { + 'project_plan': self.clone(ProjectPlanTask) + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.WBS_LEVEL1_RAW), + 'clean': self.local_target(FilenameEnum.WBS_LEVEL1), + 'project_title': self.local_target(FilenameEnum.WBS_LEVEL1_PROJECT_TITLE) + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Creating Work Breakdown Structure (WBS) Level 1...") + + # Read the project plan JSON from the dependency. + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + + # Build the query using the project plan. + query = format_json_for_use_in_query(project_plan_dict) + + # Execute the WBS Level 1 creation. + create_wbs_level1 = CreateWBSLevel1.execute(llm, query) + + # Save the raw output. + wbs_level1_raw_dict = create_wbs_level1.raw_response_dict() + with self.output()['raw'].open("w") as f: + json.dump(wbs_level1_raw_dict, f, indent=2) + + # Save the cleaned up result. + wbs_level1_result_json = create_wbs_level1.cleanedup_dict() + with self.output()['clean'].open("w") as f: + json.dump(wbs_level1_result_json, f, indent=2) + + # Save the project title. + with self.output()['project_title'].open("w") as f: + f.write(create_wbs_level1.project_title) + + logger.info("WBS Level 1 created successfully.") diff --git a/worker_plan/worker_plan_internal/plan/stages/create_wbs_level2.py b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level2.py new file mode 100644 index 000000000..6ae31ff5f --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level2.py @@ -0,0 +1,78 @@ +"""CreateWBSLevel2Task - Creates the Work Breakdown Structure (WBS) Level 2.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.create_wbs_level2 import CreateWBSLevel2 +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask + +logger = logging.getLogger(__name__) + + +class CreateWBSLevel2Task(PlanTask): + """ + Creates the Work Breakdown Structure (WBS) Level 2. + Depends on: + - ProjectPlanTask: provides the project plan as JSON. + - CreateWBSLevel1Task: provides the cleaned WBS Level 1 result. + Produces: + - Raw WBS Level 2 output (007-wbs_level2_raw.json) + - Cleaned WBS Level 2 output (008-wbs_level2.json) + """ + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'wbs_level1': self.clone(CreateWBSLevel1Task), + 'data_collection': self.clone(DataCollectionTask), + } + + def output(self): + return { + 'raw': self.local_target(FilenameEnum.WBS_LEVEL2_RAW), + 'clean': self.local_target(FilenameEnum.WBS_LEVEL2) + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Creating Work Breakdown Structure (WBS) Level 2...") + + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['wbs_level1']['clean'].open("r") as f: + wbs_level1_result_json = json.load(f) + + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'project_plan.md':\n{project_plan_markdown}\n\n" + f"File 'WBS Level 1.json':\n{format_json_for_use_in_query(wbs_level1_result_json)}\n\n" + f"File 'data_collection.md':\n{data_collection_markdown}" + ) + + # Execute the WBS Level 2 creation. + create_wbs_level2 = CreateWBSLevel2.execute(llm, query) + + # Retrieve and write the raw output. + wbs_level2_raw_dict = create_wbs_level2.raw_response_dict() + with self.output()['raw'].open("w") as f: + json.dump(wbs_level2_raw_dict, f, indent=2) + + # Retrieve and write the cleaned output (e.g. major phases with subtasks). + with self.output()['clean'].open("w") as f: + json.dump(create_wbs_level2.major_phases_with_subtasks, f, indent=2) + + logger.info("WBS Level 2 created successfully.") diff --git a/worker_plan/worker_plan_internal/plan/stages/create_wbs_level3.py b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level3.py new file mode 100644 index 000000000..282e94cc5 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/create_wbs_level3.py @@ -0,0 +1,136 @@ +"""CreateWBSLevel3Task - Creates the Work Breakdown Structure (WBS) Level 3.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.create_wbs_level3 import CreateWBSLevel3 +from worker_plan_internal.wbs.wbs_task import WBSProject +from worker_plan_internal.wbs.wbs_populate import WBSPopulate +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task +from worker_plan_internal.plan.stages.estimate_task_durations import EstimateTaskDurationsTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask + +logger = logging.getLogger(__name__) + + +class CreateWBSLevel3Task(PlanTask): + """ + This task creates the Work Breakdown Structure (WBS) Level 3, by decomposing tasks from Level 2 into subtasks. + + It depends on: + - ProjectPlanTask: provides the project plan JSON. + - WBSProjectLevel1AndLevel2Task: provides the major phases with subtasks and the task UUIDs. + - EstimateTaskDurationsTask: provides the aggregated task durations (task_duration_list). + + For each task without any subtasks, a query is built and executed using the LLM. + The raw JSON result for each task is written to a file using the template from FilenameEnum. + Finally, all individual results are accumulated and written as an aggregated JSON file. + """ + def output(self): + return self.local_target(FilenameEnum.WBS_LEVEL3) + + def requires(self): + return { + 'project_plan': self.clone(ProjectPlanTask), + 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), + 'task_durations': self.clone(EstimateTaskDurationsTask), + 'data_collection': self.clone(DataCollectionTask), + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + logger.info("Creating Work Breakdown Structure (WBS) Level 3...") + + # Read inputs from required tasks. + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + + with self.input()['wbs_project'].open("r") as f: + wbs_project_dict = json.load(f) + wbs_project = WBSProject.from_dict(wbs_project_dict) + + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + + # Load the estimated task durations. + task_duration_list_path = self.input()['task_durations'].path + WBSPopulate.extend_project_with_durations_json(wbs_project, task_duration_list_path) + + # for each task in the wbs_project, find the task that has no children + tasks_with_no_children = [] + def visit_task(task): + if len(task.task_children) == 0: + tasks_with_no_children.append(task) + else: + for child in task.task_children: + visit_task(child) + visit_task(wbs_project.root_task) + + # for each task with no children, extract the task_id + decompose_task_id_list = [] + for task in tasks_with_no_children: + decompose_task_id_list.append(task.id) + + logger.info("There are %d tasks to be decomposed.", len(decompose_task_id_list)) + + # In production mode, all chunks are processed. + # In developer mode, truncate to only 2 chunks for fast turnaround cycle. Otherwise LOTS of tasks are to be decomposed. + logger.info(f"CreateWBSLevel3Task.speedvsdetail: {self.speedvsdetail}") + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") + decompose_task_id_list = decompose_task_id_list[:2] + else: + logger.info("Processing all chunks.") + + project_plan_str = format_json_for_use_in_query(project_plan_dict) + wbs_project_str = format_json_for_use_in_query(wbs_project.to_dict()) + + # Loop over each task ID. + wbs_level3_result_accumulated = [] + total_tasks = len(decompose_task_id_list) + for index, task_id in enumerate(decompose_task_id_list, start=1): + logger.info("Decomposing task %d of %d", index, total_tasks) + + query = ( + f"The project plan:\n{project_plan_str}\n\n" + f"Data collection:\n{data_collection_markdown}\n\n" + f"Work breakdown structure:\n{wbs_project_str}\n\n" + f"Only decompose this task:\n\"{task_id}\"" + ) + + # IDEA: If the chunk file already exist, then there is no need to run the LLM again. + def execute_create_wbs_level3(llm: LLM) -> CreateWBSLevel3: + return CreateWBSLevel3.execute(llm, query, task_id) + + try: + create_wbs_level3 = llm_executor.run(execute_create_wbs_level3) + except PipelineStopRequested: + # Re-raise PipelineStopRequested without wrapping it + raise + except Exception as e: + logger.error(f"WBS Level 3 task {index} LLM interaction failed.", exc_info=True) + raise ValueError(f"WBS Level 3 task {index} LLM interaction failed.") from e + + wbs_level3_raw_dict = create_wbs_level3.raw_response_dict() + + # Write the raw JSON for this task using the FilenameEnum template. + raw_filename = FilenameEnum.WBS_LEVEL3_RAW_TEMPLATE.value.format(index) + raw_chunk_path = self.run_id_dir / raw_filename + with open(raw_chunk_path, 'w') as f: + json.dump(wbs_level3_raw_dict, f, indent=2) + + # Accumulate the decomposed tasks. + wbs_level3_result_accumulated.extend(create_wbs_level3.tasks) + + # Write the aggregated WBS Level 3 result. + aggregated_path = self.file_path(FilenameEnum.WBS_LEVEL3) + with open(aggregated_path, 'w') as f: + json.dump(wbs_level3_result_accumulated, f, indent=2) + + logger.info("WBS Level 3 created and aggregated results written to %s", aggregated_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/estimate_task_durations.py b/worker_plan/worker_plan_internal/plan/stages/estimate_task_durations.py new file mode 100644 index 000000000..c6cd991f8 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/estimate_task_durations.py @@ -0,0 +1,117 @@ +"""EstimateTaskDurationsTask - Estimates durations for WBS tasks in chunks.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.estimate_wbs_task_durations import EstimateWBSTaskDurations +from worker_plan_internal.wbs.wbs_task import WBSProject +from worker_plan_internal.llm_util.llm_executor import LLMExecutor, PipelineStopRequested +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task + +logger = logging.getLogger(__name__) + + +class EstimateTaskDurationsTask(PlanTask): + """ + This task estimates durations for WBS tasks in chunks. + + It depends on: + - ProjectPlanTask: providing the project plan JSON. + - WBSProjectLevel1AndLevel2Task: providing the major phases with subtasks and the task UUIDs. + + For each chunk of 3 task IDs, a raw JSON file (e.g. "011-1-task_durations_raw.json") is written, + and an aggregated JSON file (defined by FilenameEnum.TASK_DURATIONS) is produced. + + IDEA: 1st estimate the Tasks that have zero children. + 2nd estimate tasks that have children where all children have been estimated. + repeat until all tasks have been estimated. + """ + def output(self): + return self.local_target(FilenameEnum.TASK_DURATIONS) + + def requires(self): + return { + 'project_plan': self.clone(ProjectPlanTask), + 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + logger.info("Estimating task durations...") + + # Load the project plan JSON. + with self.input()['project_plan']['raw'].open("r") as f: + project_plan_dict = json.load(f) + + with self.input()['wbs_project'].open("r") as f: + wbs_project_dict = json.load(f) + wbs_project = WBSProject.from_dict(wbs_project_dict) + + # json'ish representation of the major phases in the WBS, and their subtasks. + root_task = wbs_project.root_task + major_tasks = [child.to_dict() for child in root_task.task_children] + major_phases_with_subtasks = major_tasks + + # Don't include uuid of the root task. It's the child tasks that are of interest to estimate. + decompose_task_id_list = [] + for task in wbs_project.root_task.task_children: + decompose_task_id_list.extend(task.task_ids()) + + logger.info(f"There are {len(decompose_task_id_list)} tasks to be estimated.") + + # Split the task IDs into chunks of 3. + task_ids_chunks = [decompose_task_id_list[i:i + 3] for i in range(0, len(decompose_task_id_list), 3)] + + # In production mode, all chunks are processed. + # In developer mode, truncate to only 2 chunks for fast turnaround cycle. Otherwise LOTS of tasks are to be estimated. + logger.info(f"EstimateTaskDurationsTask.speedvsdetail: {self.speedvsdetail}") + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") + task_ids_chunks = task_ids_chunks[:2] + else: + logger.info("Processing all chunks.") + + # Process each chunk. + accumulated_task_duration_list = [] + for index, task_ids_chunk in enumerate(task_ids_chunks, start=1): + logger.info("Processing chunk %d of %d", index, len(task_ids_chunks)) + + query = EstimateWBSTaskDurations.format_query( + project_plan_dict, + major_phases_with_subtasks, + task_ids_chunk + ) + + # IDEA: If the chunk file already exist, then there is no need to run the LLM again. + def execute_estimate_task_durations(llm: LLM) -> EstimateWBSTaskDurations: + return EstimateWBSTaskDurations.execute(llm, query) + + try: + estimate_durations = llm_executor.run(execute_estimate_task_durations) + except PipelineStopRequested: + # Re-raise PipelineStopRequested without wrapping it + raise + except Exception as e: + logger.error(f"Task durations chunk {index} LLM interaction failed.", exc_info=True) + raise ValueError(f"Task durations chunk {index} LLM interaction failed.") from e + + durations_raw_dict = estimate_durations.raw_response_dict() + + # Write the raw JSON for this chunk. + filename = FilenameEnum.TASK_DURATIONS_RAW_TEMPLATE.format(index) + raw_chunk_path = self.run_id_dir / filename + with open(raw_chunk_path, "w") as f: + json.dump(durations_raw_dict, f, indent=2) + + accumulated_task_duration_list.extend(durations_raw_dict.get('task_details', [])) + + # Write the aggregated task durations. + aggregated_path = self.file_path(FilenameEnum.TASK_DURATIONS) + with open(aggregated_path, "w") as f: + json.dump(accumulated_task_duration_list, f, indent=2) + + logger.info("Task durations estimated and aggregated results written to %s", aggregated_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/identify_task_dependencies.py b/worker_plan/worker_plan_internal/plan/stages/identify_task_dependencies.py new file mode 100644 index 000000000..a2268f1b4 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/identify_task_dependencies.py @@ -0,0 +1,66 @@ +"""IdentifyTaskDependenciesTask - Identifies the dependencies between WBS tasks.""" +import json +import logging +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.identify_wbs_task_dependencies import IdentifyWBSTaskDependencies +from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.create_wbs_level2 import CreateWBSLevel2Task +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask + +logger = logging.getLogger(__name__) + + +class IdentifyTaskDependenciesTask(PlanTask): + """ + This task identifies the dependencies between WBS tasks. + """ + def output(self): + return self.local_target(FilenameEnum.TASK_DEPENDENCIES_RAW) + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'wbs_level2': self.clone(CreateWBSLevel2Task), + 'data_collection': self.clone(DataCollectionTask), + } + + def run_with_llm(self, llm: LLM) -> None: + logger.info("Identifying task dependencies...") + + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['wbs_level2']['clean'].open("r") as f: + major_phases_with_subtasks = json.load(f) + + # Build the query + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'project_plan.md':\n{project_plan_markdown}\n\n" + f"File 'Work Breakdown Structure.json':\n{format_json_for_use_in_query(major_phases_with_subtasks)}\n\n" + f"File 'data_collection.md':\n{data_collection_markdown}" + ) + + # Execute the dependency identification. + identify_dependencies = IdentifyWBSTaskDependencies.execute(llm, query) + dependencies_raw_dict = identify_dependencies.raw_response_dict() + + # Write the raw dependencies JSON to the output file. + with self.output().open("w") as f: + json.dump(dependencies_raw_dict, f, indent=2) + + logger.info("Task dependencies identified and written to %s", self.output().path) diff --git a/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_and_level2.py b/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_and_level2.py new file mode 100644 index 000000000..e9a604636 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_and_level2.py @@ -0,0 +1,35 @@ +"""WBSProjectLevel1AndLevel2Task - Create a WBS project from Level 1 and Level 2 JSON files.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.wbs.wbs_populate import WBSPopulate +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.create_wbs_level2 import CreateWBSLevel2Task + + +class WBSProjectLevel1AndLevel2Task(PlanTask): + """ + Create a WBS project from the WBS Level 1 and Level 2 JSON files. + + It depends on: + - CreateWBSLevel1Task: providing the cleaned WBS Level 1 JSON. + - CreateWBSLevel2Task: providing the major phases with subtasks and the task UUIDs. + """ + def output(self): + return self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2) + + def requires(self): + return { + 'wbs_level1': self.clone(CreateWBSLevel1Task), + 'wbs_level2': self.clone(CreateWBSLevel2Task), + } + + def run_inner(self): + wbs_level1_path = self.input()['wbs_level1']['clean'].path + wbs_level2_path = self.input()['wbs_level2']['clean'].path + wbs_project = WBSPopulate.project_from_level1_json(wbs_level1_path) + WBSPopulate.extend_project_with_level2_json(wbs_project, wbs_level2_path) + + json_representation = json.dumps(wbs_project.to_dict(), indent=2) + with self.output().open("w") as f: + f.write(json_representation) diff --git a/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_level2_level3.py b/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_level2_level3.py new file mode 100644 index 000000000..033075eda --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/wbs_project_level1_level2_level3.py @@ -0,0 +1,46 @@ +"""WBSProjectLevel1AndLevel2AndLevel3Task - Create a WBS project from Level 1, Level 2 and Level 3 JSON files.""" +import json +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.wbs.wbs_task import WBSProject +from worker_plan_internal.wbs.wbs_populate import WBSPopulate +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task +from worker_plan_internal.plan.stages.create_wbs_level3 import CreateWBSLevel3Task + + +class WBSProjectLevel1AndLevel2AndLevel3Task(PlanTask): + """ + Create a WBS project from the WBS Level 1 and Level 2 and Level 3 JSON files. + + It depends on: + - WBSProjectLevel1AndLevel2Task: providing the major phases with subtasks and the task UUIDs. + - CreateWBSLevel3Task: providing the decomposed tasks. + """ + def output(self): + return { + 'full': self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2_AND_LEVEL3_FULL), + 'csv': self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2_AND_LEVEL3_CSV) + } + + def requires(self): + return { + 'wbs_project12': self.clone(WBSProjectLevel1AndLevel2Task), + 'wbs_level3': self.clone(CreateWBSLevel3Task), + } + + def run_inner(self): + wbs_project_path = self.input()['wbs_project12'].path + with open(wbs_project_path, "r") as f: + wbs_project_dict = json.load(f) + wbs_project = WBSProject.from_dict(wbs_project_dict) + + wbs_level3_path = self.input()['wbs_level3'].path + WBSPopulate.extend_project_with_decomposed_tasks_json(wbs_project, wbs_level3_path) + + json_representation = json.dumps(wbs_project.to_dict(), indent=2) + with self.output()['full'].open("w") as f: + f.write(json_representation) + + csv_representation = wbs_project.to_csv_string() + with self.output()['csv'].open("w") as f: + f.write(csv_representation) From f8c2f23a942b08183b28e18b65af65939a1d1c94 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 19:06:30 +0200 Subject: [PATCH 09/12] refactor: extract Phase 12-13 pipeline stages (schedule, review & report) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/stages/create_schedule.py | 112 ++++++++++++++++ .../plan/stages/executive_summary.py | 96 +++++++++++++ .../plan/stages/premortem.py | 104 +++++++++++++++ .../plan/stages/questions_and_answers.py | 100 ++++++++++++++ .../plan/stages/report.py | 95 +++++++++++++ .../plan/stages/review_plan.py | 93 +++++++++++++ .../plan/stages/self_audit.py | 126 ++++++++++++++++++ 7 files changed, 726 insertions(+) create mode 100644 worker_plan/worker_plan_internal/plan/stages/create_schedule.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/executive_summary.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/premortem.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/questions_and_answers.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/report.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/review_plan.py create mode 100644 worker_plan/worker_plan_internal/plan/stages/self_audit.py diff --git a/worker_plan/worker_plan_internal/plan/stages/create_schedule.py b/worker_plan/worker_plan_internal/plan/stages/create_schedule.py new file mode 100644 index 000000000..820778222 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/create_schedule.py @@ -0,0 +1,112 @@ +"""CreateScheduleTask - Builds the project schedule and exports Gantt charts.""" +import json +from datetime import date, datetime +from typing import Any +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.schedule.project_schedule_populator import ProjectSchedulePopulator +from worker_plan_internal.schedule.schedule import ProjectSchedule +from worker_plan_internal.schedule.export_gantt_dhtmlx import ExportGanttDHTMLX +from worker_plan_internal.schedule.export_gantt_csv import ExportGanttCSV +from worker_plan_internal.wbs.wbs_task import WBSProject +from worker_plan_internal.wbs.wbs_task_tooltip import WBSTaskTooltip +from worker_plan_internal.plan.pipeline_config import PIPELINE_CONFIG +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.start_time import StartTimeTask +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.identify_task_dependencies import IdentifyTaskDependenciesTask +from worker_plan_internal.plan.stages.estimate_task_durations import EstimateTaskDurationsTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task + + +class CreateScheduleTask(PlanTask): + def output(self): + return { + 'dhtmlx_html': self.local_target(FilenameEnum.SCHEDULE_GANTT_DHTMLX_HTML), + 'machai_csv': self.local_target(FilenameEnum.SCHEDULE_GANTT_MACHAI_CSV) + } + + def requires(self): + return { + 'start_time': self.clone(StartTimeTask), + 'wbs_level1': self.clone(CreateWBSLevel1Task), + 'dependencies': self.clone(IdentifyTaskDependenciesTask), + 'durations': self.clone(EstimateTaskDurationsTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task) + } + + def run_inner(self): + # For the report title, use the 'project_title' of the WBS Level 1 result. + with self.input()['wbs_level1']['project_title'].open("r") as f: + title = f.read() + with self.input()['dependencies'].open("r") as f: + dependencies_dict = json.load(f) + with self.input()['durations'].open("r") as f: + duration_list: list[dict[str, Any]] = json.load(f) + wbs_project_path = self.input()['wbs_project123']['full'].path + with open(wbs_project_path, "r") as f: + wbs_project_dict = json.load(f) + wbs_project = WBSProject.from_dict(wbs_project_dict) + + # Read the start time from the StartTimeTask to get the actual pipeline start date + with self.input()['start_time'].open("r") as f: + start_time_dict = json.load(f) + + # The start_time.server_iso_utc is in format "YYYY-MM-DDTHH:MM:SSZ" + utc_timestamp = start_time_dict.get('server_iso_utc') or start_time_dict.get('utc_timestamp', '') + # The 'Z' suffix for UTC is not supported by fromisoformat() in Python < 3.11. Replace, ensures compatibility. + project_start_dt: datetime = datetime.fromisoformat(utc_timestamp.replace('Z', '+00:00')) + project_start: date = project_start_dt.date() + + # logger.debug(f"dependencies_dict {dependencies_dict}") + # logger.debug(f"duration_list {duration_list}") + # logger.debug(f"wbs_project {wbs_project.to_dict()}") + + # Tooltips with a detailed description of each task. + task_id_to_html_tooltip_dict: dict[str, str] = WBSTaskTooltip.html_tooltips(wbs_project) + task_id_to_text_tooltip_dict: dict[str, str] = WBSTaskTooltip.text_tooltips(wbs_project) + + project_schedule: ProjectSchedule = ProjectSchedulePopulator.populate( + wbs_project=wbs_project, + duration_list=duration_list + ) + + # Export the Gantt chart to CSV. + # Always run the CSV export so that the code gets exercised, otherwise the code will rot. + csv_data: str = ExportGanttCSV.to_gantt_csv( + project_schedule=project_schedule, + project_start=project_start, + task_id_to_tooltip_dict=task_id_to_text_tooltip_dict + ) + if PIPELINE_CONFIG.enable_csv_export == False: + # When disabled, then hide the "Export to CSV" button and don't embed the CSV data in the html report. + csv_data = None + + ExportGanttCSV.save( + project_schedule=project_schedule, + path=self.output()['machai_csv'].path, + project_start=project_start, + task_id_to_tooltip_dict=task_id_to_text_tooltip_dict + ) + + # Identify the tasks that should be treated as project activities. + task_ids_to_treat_as_project_activities = wbs_project.task_ids_with_one_or_more_children() + + # Export the Gantt chart to Frappe. + # I'm disappointed by Frappe, it lacks a lot of features that are present in DHTMLX. + # ExportGanttFrappe.save( + # project_schedule=project_schedule, + # path=self.output()['frappe_html'].path, + # project_start=project_start, + # task_ids_to_treat_as_project_activities=task_ids_to_treat_as_project_activities + # ) + + # Export the Gantt chart to DHTMLX. + ExportGanttDHTMLX.save( + project_schedule=project_schedule, + path=self.output()['dhtmlx_html'].path, + project_start=project_start, + task_ids_to_treat_as_project_activities=task_ids_to_treat_as_project_activities, + task_id_to_tooltip_dict=task_id_to_html_tooltip_dict, + title=title, + csv_data=csv_data + ) diff --git a/worker_plan/worker_plan_internal/plan/stages/executive_summary.py b/worker_plan/worker_plan_internal/plan/stages/executive_summary.py new file mode 100644 index 000000000..0c7ba83bb --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/executive_summary.py @@ -0,0 +1,96 @@ +"""ExecutiveSummaryTask - Creates an executive summary of the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.executive_summary import ExecutiveSummary +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask + + +class ExecutiveSummaryTask(PlanTask): + """ + Create an executive summary of the plan. + """ + def output(self): + return { + 'raw': self.local_target(FilenameEnum.EXECUTIVE_SUMMARY_RAW), + 'markdown': self.local_target(FilenameEnum.EXECUTIVE_SUMMARY_MARKDOWN) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'data_collection': self.clone(DataCollectionTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'expert_review': self.clone(ExpertReviewTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'review_plan': self.clone(ReviewPlanTask) + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['pitch_markdown']['markdown'].open("r") as f: + pitch_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + with self.input()['wbs_project123']['csv'].open("r") as f: + wbs_project_csv = f.read() + with self.input()['review_plan']['markdown'].open("r") as f: + review_plan_markdown = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'data-collection.md':\n{data_collection_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'pitch.md':\n{pitch_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}\n\n" + f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" + f"File 'review-plan.md':\n{review_plan_markdown}" + ) + + # Create the executive summary. + executive_summary = ExecutiveSummary.execute(llm, query) + + # Save the results. + json_path = self.output()['raw'].path + executive_summary.save_raw(json_path) + markdown_path = self.output()['markdown'].path + executive_summary.save_markdown(markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/premortem.py b/worker_plan/worker_plan_internal/plan/stages/premortem.py new file mode 100644 index 000000000..cc9d9bfe7 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/premortem.py @@ -0,0 +1,104 @@ +"""PremortemTask - Performs a premortem analysis of the plan.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.diagnostics.premortem import Premortem +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask +from worker_plan_internal.plan.stages.questions_and_answers import QuestionsAndAnswersTask + + +class PremortemTask(PlanTask): + def output(self): + return { + 'raw': self.local_target(FilenameEnum.PREMORTEM_RAW), + 'markdown': self.local_target(FilenameEnum.PREMORTEM_MARKDOWN) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'consolidate_governance': self.clone(ConsolidateGovernanceTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'data_collection': self.clone(DataCollectionTask), + 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'expert_review': self.clone(ExpertReviewTask), + 'project_plan': self.clone(ProjectPlanTask), + 'review_plan': self.clone(ReviewPlanTask), + 'questions_and_answers': self.clone(QuestionsAndAnswersTask) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['pitch_markdown']['markdown'].open("r") as f: + pitch_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + with self.input()['wbs_project123']['csv'].open("r") as f: + wbs_project_csv = f.read() + with self.input()['review_plan']['markdown'].open("r") as f: + review_plan_markdown = f.read() + with self.input()['questions_and_answers']['markdown'].open("r") as f: + questions_and_answers_markdown = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'data-collection.md':\n{data_collection_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'pitch.md':\n{pitch_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}\n\n" + f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" + f"File 'review-plan.md':\n{review_plan_markdown}\n\n" + f"File 'questions-and-answers.md':\n{questions_and_answers_markdown}" + ) + + # Invoke the LLM + premortem = Premortem.execute(llm_executor=llm_executor, speed_vs_detail=self.speedvsdetail, user_prompt=query) + + # Save the results. + json_path = self.output()['raw'].path + premortem.save_raw(json_path) + markdown_path = self.output()['markdown'].path + premortem.save_markdown(markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/questions_and_answers.py b/worker_plan/worker_plan_internal/plan/stages/questions_and_answers.py new file mode 100644 index 000000000..0c7d6dfa5 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/questions_and_answers.py @@ -0,0 +1,100 @@ +"""QuestionsAndAnswersTask - Generates Q&A about the plan.""" +from llama_index.core.llms.llm import LLM +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.questions_answers.questions_answers import QuestionsAnswers +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask + + +class QuestionsAndAnswersTask(PlanTask): + def output(self): + return { + 'raw': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_RAW), + 'markdown': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_MARKDOWN), + 'html': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_HTML) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'consolidate_governance': self.clone(ConsolidateGovernanceTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'data_collection': self.clone(DataCollectionTask), + 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'expert_review': self.clone(ExpertReviewTask), + 'project_plan': self.clone(ProjectPlanTask), + 'review_plan': self.clone(ReviewPlanTask), + } + + def run_with_llm(self, llm: LLM) -> None: + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['pitch_markdown']['markdown'].open("r") as f: + pitch_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + with self.input()['wbs_project123']['csv'].open("r") as f: + wbs_project_csv = f.read() + with self.input()['review_plan']['markdown'].open("r") as f: + review_plan_markdown = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'data-collection.md':\n{data_collection_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'pitch.md':\n{pitch_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}\n\n" + f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" + f"File 'review-plan.md':\n{review_plan_markdown}" + ) + + # Invoke the LLM + question_answers = QuestionsAnswers.execute(llm, query) + + # Save the results. + json_path = self.output()['raw'].path + question_answers.save_raw(json_path) + markdown_path = self.output()['markdown'].path + question_answers.save_markdown(markdown_path) + html_path = self.output()['html'].path + question_answers.save_html(html_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/report.py b/worker_plan/worker_plan_internal/plan/stages/report.py new file mode 100644 index 000000000..d379bd1c7 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/report.py @@ -0,0 +1,95 @@ +"""ReportTask - Generates the final HTML report document.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask, REPORT_EXECUTE_PLAN_SECTION_HIDDEN +from worker_plan_internal.report.report_generator import ReportGenerator +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.redline_gate import RedlineGateTask +from worker_plan_internal.plan.stages.premise_attack import PremiseAttackTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask +from worker_plan_internal.plan.stages.executive_summary import ExecutiveSummaryTask +from worker_plan_internal.plan.stages.create_schedule import CreateScheduleTask +from worker_plan_internal.plan.stages.questions_and_answers import QuestionsAndAnswersTask +from worker_plan_internal.plan.stages.premortem import PremortemTask +from worker_plan_internal.plan.stages.self_audit import SelfAuditTask + + +class ReportTask(PlanTask): + """ + Generate a report html document. + """ + def output(self): + return self.local_target(FilenameEnum.REPORT) + + def requires(self): + return { + 'setup': self.clone(SetupTask), + 'redline_gate': self.clone(RedlineGateTask), + 'premise_attack': self.clone(PremiseAttackTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'consolidate_governance': self.clone(ConsolidateGovernanceTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'data_collection': self.clone(DataCollectionTask), + 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), + 'wbs_level1': self.clone(CreateWBSLevel1Task), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'expert_review': self.clone(ExpertReviewTask), + 'project_plan': self.clone(ProjectPlanTask), + 'review_plan': self.clone(ReviewPlanTask), + 'executive_summary': self.clone(ExecutiveSummaryTask), + 'create_schedule': self.clone(CreateScheduleTask), + 'questions_and_answers': self.clone(QuestionsAndAnswersTask), + 'premortem': self.clone(PremortemTask), + 'self_audit': self.clone(SelfAuditTask) + } + + def run_inner(self): + # For the report title, use the 'project_title' of the WBS Level 1 result. + with self.input()['wbs_level1']['project_title'].open("r") as f: + title = f.read() + + rg = ReportGenerator() + rg.append_markdown('Executive Summary', self.input()['executive_summary']['markdown'].path) + rg.append_html('Gantt Interactive', self.input()['create_schedule']['dhtmlx_html'].path) + rg.append_markdown('Pitch', self.input()['pitch_markdown']['markdown'].path) + rg.append_markdown('Project Plan', self.input()['project_plan']['markdown'].path) + rg.append_markdown('Strategic Decisions', self.input()['strategic_decisions_markdown']['markdown'].path) + rg.append_markdown('Scenarios', self.input()['scenarios_markdown']['markdown'].path) + rg.append_markdown('Assumptions', self.input()['consolidate_assumptions_markdown']['full'].path) + rg.append_markdown('Governance', self.input()['consolidate_governance'].path) + rg.append_markdown('Related Resources', self.input()['related_resources']['markdown'].path) + rg.append_markdown('Data Collection', self.input()['data_collection']['markdown'].path) + rg.append_markdown('Documents to Create and Find', self.input()['documents_to_create_and_find'].path) + rg.append_markdown('SWOT Analysis', self.input()['swot_analysis']['markdown'].path) + rg.append_markdown('Team', self.input()['team_markdown'].path) + rg.append_markdown('Expert Criticism', self.input()['expert_review'].path) + rg.append_csv('Work Breakdown Structure', self.input()['wbs_project123']['csv'].path) + rg.append_markdown('Review Plan', self.input()['review_plan']['markdown'].path) + rg.append_html('Questions & Answers', self.input()['questions_and_answers']['html'].path) + rg.append_markdown_with_tables('Premortem', self.input()['premortem']['markdown'].path) + rg.append_markdown_with_tables('Self Audit', self.input()['self_audit']['markdown'].path) + rg.append_initial_prompt_vetted( + document_title='Initial Prompt Vetted', + initial_prompt_file_path=self.input()['setup'].path, + redline_gate_markdown_file_path=self.input()['redline_gate']['markdown'].path, + premise_attack_markdown_file_path=self.input()['premise_attack']['markdown'].path + ) + rg.save_report(self.output().path, title=title, execute_plan_section_hidden=REPORT_EXECUTE_PLAN_SECTION_HIDDEN) diff --git a/worker_plan/worker_plan_internal/plan/stages/review_plan.py b/worker_plan/worker_plan_internal/plan/stages/review_plan.py new file mode 100644 index 000000000..66bd44b1f --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/review_plan.py @@ -0,0 +1,93 @@ +"""ReviewPlanTask - Asks questions about the almost finished plan.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.plan.review_plan import ReviewPlan +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task + + +class ReviewPlanTask(PlanTask): + """ + Ask questions about the almost finished plan. + """ + def output(self): + return { + 'raw': self.local_target(FilenameEnum.REVIEW_PLAN_RAW), + 'markdown': self.local_target(FilenameEnum.REVIEW_PLAN_MARKDOWN) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'project_plan': self.clone(ProjectPlanTask), + 'data_collection': self.clone(DataCollectionTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'expert_review': self.clone(ExpertReviewTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['pitch_markdown']['markdown'].open("r") as f: + pitch_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + with self.input()['wbs_project123']['csv'].open("r") as f: + wbs_project_csv = f.read() + + # Build the query. + query = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'data-collection.md':\n{data_collection_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'pitch.md':\n{pitch_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}\n\n" + f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}" + ) + + # Perform the review. + review_plan = ReviewPlan.execute(llm_executor=llm_executor, document=query, speed_vs_detail=self.speedvsdetail) + + # Save the results. + json_path = self.output()['raw'].path + review_plan.save_raw(json_path) + markdown_path = self.output()['markdown'].path + review_plan.save_markdown(markdown_path) diff --git a/worker_plan/worker_plan_internal/plan/stages/self_audit.py b/worker_plan/worker_plan_internal/plan/stages/self_audit.py new file mode 100644 index 000000000..9e2d32527 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/self_audit.py @@ -0,0 +1,126 @@ +"""SelfAuditTask - Performs a self-audit of the plan.""" +import logging +from typing import Optional +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_internal.self_audit.self_audit import SelfAudit +from worker_plan_internal.llm_util.llm_executor import LLMExecutor +from worker_plan_api.speedvsdetail import SpeedVsDetailEnum +from worker_plan_api.filenames import FilenameEnum +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask +from worker_plan_internal.plan.stages.questions_and_answers import QuestionsAndAnswersTask +from worker_plan_internal.plan.stages.premortem import PremortemTask + +logger = logging.getLogger(__name__) + + +class SelfAuditTask(PlanTask): + def output(self): + return { + 'raw': self.local_target(FilenameEnum.SELF_AUDIT_RAW), + 'markdown': self.local_target(FilenameEnum.SELF_AUDIT_MARKDOWN) + } + + def requires(self): + return { + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'consolidate_governance': self.clone(ConsolidateGovernanceTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'data_collection': self.clone(DataCollectionTask), + 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'expert_review': self.clone(ExpertReviewTask), + 'project_plan': self.clone(ProjectPlanTask), + 'review_plan': self.clone(ReviewPlanTask), + 'questions_and_answers': self.clone(QuestionsAndAnswersTask), + 'premortem': self.clone(PremortemTask) + } + + def run_inner(self): + llm_executor: LLMExecutor = self.create_llm_executor() + + # Read inputs from required tasks. + with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: + strategic_decisions_markdown = f.read() + with self.input()['scenarios_markdown']['markdown'].open("r") as f: + scenarios_markdown = f.read() + with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: + assumptions_markdown = f.read() + with self.input()['project_plan']['markdown'].open("r") as f: + project_plan_markdown = f.read() + with self.input()['data_collection']['markdown'].open("r") as f: + data_collection_markdown = f.read() + with self.input()['related_resources']['markdown'].open("r") as f: + related_resources_markdown = f.read() + with self.input()['swot_analysis']['markdown'].open("r") as f: + swot_analysis_markdown = f.read() + with self.input()['team_markdown'].open("r") as f: + team_markdown = f.read() + with self.input()['pitch_markdown']['markdown'].open("r") as f: + pitch_markdown = f.read() + with self.input()['expert_review'].open("r") as f: + expert_review = f.read() + with self.input()['wbs_project123']['csv'].open("r") as f: + wbs_project_csv = f.read() + with self.input()['review_plan']['markdown'].open("r") as f: + review_plan_markdown = f.read() + with self.input()['questions_and_answers']['markdown'].open("r") as f: + questions_and_answers_markdown = f.read() + with self.input()['premortem']['markdown'].open("r") as f: + premortem_markdown = f.read() + + # Build the query. + user_prompt = ( + f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" + f"File 'scenarios.md':\n{scenarios_markdown}\n\n" + f"File 'assumptions.md':\n{assumptions_markdown}\n\n" + f"File 'project-plan.md':\n{project_plan_markdown}\n\n" + f"File 'data-collection.md':\n{data_collection_markdown}\n\n" + f"File 'related-resources.md':\n{related_resources_markdown}\n\n" + f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" + f"File 'team.md':\n{team_markdown}\n\n" + f"File 'pitch.md':\n{pitch_markdown}\n\n" + f"File 'expert-review.md':\n{expert_review}\n\n" + f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" + f"File 'review-plan.md':\n{review_plan_markdown}\n\n" + f"File 'questions-and-answers.md':\n{questions_and_answers_markdown}\n\n" + f"File 'premortem.md':\n{premortem_markdown}" + ) + + logger.info(f"SelfAuditTask.speedvsdetail: {self.speedvsdetail}") + max_number_of_items: Optional[int] = None + if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: + logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 items for testing a subset of the SelfAudit items.") + max_number_of_items = 2 + else: + logger.info("Processing all SelfAudit items.") + + # Invoke the LLM + self_audit = SelfAudit.execute( + llm_executor=llm_executor, + user_prompt=user_prompt, + max_number_of_items=max_number_of_items, + ) + + # Save the results. + json_path = self.output()['raw'].path + self_audit.save_raw(json_path) + markdown_path = self.output()['markdown'].path + self_audit.save_markdown(markdown_path) From 88f38dc268a5ccb622dd6f359a165c6ac7680119 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 19:16:39 +0200 Subject: [PATCH 10/12] refactor: slim run_plan_pipeline.py to framework-only, wire FullPlanPipeline from stages Remove all ~66 task class definitions from run_plan_pipeline.py (3761 lines deleted) and create stages/full_plan_pipeline.py as the pipeline orchestrator that imports all task classes from individual stage files. ExecutePipeline.setup() now uses a deferred import of FullPlanPipeline to avoid circular dependencies. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plan/run_plan_pipeline.py | 3828 +---------------- .../plan/stages/full_plan_pipeline.py | 156 + 2 files changed, 223 insertions(+), 3761 deletions(-) create mode 100644 worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py diff --git a/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py b/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py index 4739f5ae5..8f906a9e9 100644 --- a/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py +++ b/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py @@ -17,81 +17,13 @@ from pathlib import Path import sys from llama_index.core.llms.llm import LLM -from worker_plan_internal.diagnostics.redline_gate import RedlineGate -from worker_plan_internal.diagnostics.premise_attack import PremiseAttack -from worker_plan_internal.diagnostics.premortem import Premortem -from worker_plan_internal.plan.pipeline_config import PIPELINE_CONFIG -from worker_plan_internal.lever.deduplicate_levers import DeduplicateLevers -from worker_plan_internal.lever.scenarios_markdown import ScenariosMarkdown -from worker_plan_internal.lever.strategic_decisions_markdown import StrategicDecisionsMarkdown from worker_plan_api.filenames import FilenameEnum, ExtraFilenameEnum from worker_plan_api.pipeline_version import PIPELINE_VERSION from worker_plan_api.speedvsdetail import SpeedVsDetailEnum from worker_plan_internal.utils.planexe_llmconfig import PlanExeLLMConfig -from worker_plan_internal.assume.identify_purpose import IdentifyPurpose -from worker_plan_internal.assume.identify_plan_type import IdentifyPlanType -from worker_plan_internal.assume.physical_locations import PhysicalLocations -from worker_plan_internal.assume.currency_strategy import CurrencyStrategy -from worker_plan_internal.assume.identify_risks import IdentifyRisks -from worker_plan_internal.assume.make_assumptions import MakeAssumptions -from worker_plan_internal.assume.distill_assumptions import DistillAssumptions -from worker_plan_internal.assume.review_assumptions import ReviewAssumptions -from worker_plan_internal.assume.shorten_markdown import ShortenMarkdown -from worker_plan_internal.expert.pre_project_assessment import PreProjectAssessment -from worker_plan_internal.plan.project_plan import ProjectPlan -from worker_plan_internal.document.identify_documents import IdentifyDocuments -from worker_plan_internal.document.filter_documents_to_find import FilterDocumentsToFind -from worker_plan_internal.document.filter_documents_to_create import FilterDocumentsToCreate -from worker_plan_internal.document.draft_document_to_find import DraftDocumentToFind -from worker_plan_internal.document.draft_document_to_create import DraftDocumentToCreate -from worker_plan_internal.document.markdown_with_document import markdown_rows_with_document_to_create, markdown_rows_with_document_to_find -from worker_plan_internal.governance.governance_phase1_audit import GovernancePhase1Audit -from worker_plan_internal.governance.governance_phase2_bodies import GovernancePhase2Bodies -from worker_plan_internal.governance.governance_phase3_impl_plan import GovernancePhase3ImplPlan -from worker_plan_internal.governance.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrix -from worker_plan_internal.governance.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgress -from worker_plan_internal.governance.governance_phase6_extra import GovernancePhase6Extra -from worker_plan_internal.plan.related_resources import RelatedResources -from worker_plan_internal.questions_answers.questions_answers import QuestionsAnswers -from worker_plan_internal.lever.identify_potential_levers import IdentifyPotentialLevers -from worker_plan_internal.lever.enrich_potential_levers import EnrichPotentialLevers -from worker_plan_internal.lever.focus_on_vital_few_levers import FocusOnVitalFewLevers -from worker_plan_internal.lever.candidate_scenarios import CandidateScenarios -from worker_plan_internal.lever.select_scenario import SelectScenario -from worker_plan_internal.swot.swot_analysis import SWOTAnalysis -from worker_plan_internal.expert.expert_finder import ExpertFinder -from worker_plan_internal.expert.expert_criticism import ExpertCriticism -from worker_plan_internal.expert.expert_orchestrator import ExpertOrchestrator -from worker_plan_internal.plan.create_wbs_level1 import CreateWBSLevel1 -from worker_plan_internal.plan.create_wbs_level2 import CreateWBSLevel2 -from worker_plan_internal.plan.create_wbs_level3 import CreateWBSLevel3 -from worker_plan_internal.pitch.create_pitch import CreatePitch -from worker_plan_internal.pitch.convert_pitch_to_markdown import ConvertPitchToMarkdown -from worker_plan_internal.plan.identify_wbs_task_dependencies import IdentifyWBSTaskDependencies -from worker_plan_internal.plan.estimate_wbs_task_durations import EstimateWBSTaskDurations -from worker_plan_internal.plan.data_collection import DataCollection -from worker_plan_internal.plan.review_plan import ReviewPlan -from worker_plan_internal.plan.executive_summary import ExecutiveSummary -from worker_plan_internal.team.find_team_members import FindTeamMembers -from worker_plan_internal.team.enrich_team_members_with_contract_type import EnrichTeamMembersWithContractType -from worker_plan_internal.team.enrich_team_members_with_background_story import EnrichTeamMembersWithBackgroundStory -from worker_plan_internal.team.enrich_team_members_with_environment_info import EnrichTeamMembersWithEnvironmentInfo -from worker_plan_internal.team.team_markdown_document import TeamMarkdownDocumentBuilder -from worker_plan_internal.team.review_team import ReviewTeam -from worker_plan_internal.self_audit.self_audit import SelfAudit -from worker_plan_internal.wbs.wbs_task import WBSTask, WBSProject -from worker_plan_internal.wbs.wbs_populate import WBSPopulate -from worker_plan_internal.wbs.wbs_task_tooltip import WBSTaskTooltip -from worker_plan_internal.schedule.project_schedule_populator import ProjectSchedulePopulator -from worker_plan_internal.schedule.schedule import ProjectSchedule -from worker_plan_internal.schedule.export_gantt_dhtmlx import ExportGanttDHTMLX -from worker_plan_internal.schedule.export_gantt_csv import ExportGanttCSV -# from worker_plan_internal.schedule.export_gantt_mermaid import ExportGanttMermaid from worker_plan_internal.llm_util.llm_executor import LLMExecutor, LLMModelFromName, ShouldStopCallbackParameters, PipelineStopRequested, RetryConfig from worker_plan_internal.llm_factory import get_llm_names_by_priority, SPECIAL_AUTO_ID, is_valid_llm_name from worker_plan_api.model_profile import ModelProfileEnum, normalize_model_profile -from worker_plan_internal.format_json_for_use_in_query import format_json_for_use_in_query -from worker_plan_internal.report.report_generator import ReportGenerator from worker_plan_internal.luigi_util.obtain_output_files import ObtainOutputFiles from worker_plan_internal.plan.pipeline_environment import PipelineEnvironment from worker_plan_internal.plan.ping_llm import run_ping_llm_report @@ -148,3704 +80,77 @@ class PlanTask(luigi.Task): def file_path(self, filename: FilenameEnum) -> Path: return self.run_id_dir / filename.value - - def local_target(self, filename: FilenameEnum) -> luigi.LocalTarget: - return luigi.LocalTarget(self.file_path(filename)) - - def create_llm_executor(self) -> LLMExecutor: - """ - Create an LLMExecutor instance. - - Responsible for stopping the pipeline when the user presses Ctrl-C or closes the browser tab. - - Fallback mechanism to try the next LLM if the current one fails. - """ - # Redirect the callback to the pipeline_executor_callback. - def should_stop_callback(parameters: ShouldStopCallbackParameters) -> None: - if self._pipeline_executor_callback is None: - return - # The pipeline_executor_callback expects (task, duration) but we have ShouldStopCallbackParameters - total_duration = parameters.total_duration - self._pipeline_executor_callback(self, total_duration) - - llm_model_instances = LLMModelFromName.from_names(self.llm_models) - - return LLMExecutor( - llm_models=llm_model_instances, - should_stop_callback=should_stop_callback, - retry_config=RetryConfig(), - ) - - def run(self): - """ - Don't override this method. Instead either override the run_inner() method, or override the run_with_llm() method. - """ - try: - self.run_inner() - except PipelineStopRequested as e: - logger.debug(f"{self.__class__.__name__} -> PipelineStopRequested raised: {e}") - # This exception is raised by the should_stop_callback - # If we get here, it means that the pipeline was aborted by the callback, such as by the user pressing Ctrl-C or closing the browser tab. - # Create a flag file to signal that the stop was intentional. - flag_path = self.run_id_dir / ExtraFilenameEnum.PIPELINE_STOP_REQUESTED_FLAG.value - logger.info(f"Creating stop flag file: {flag_path!r}") - flag_path.touch() - raise - except Exception as e: - # Re-raise the exception with a more descriptive message - raise Exception(f"Failed to run {self.__class__.__name__} with any of the LLMs in the list: {self.llm_models!r} for run_id_dir: {self.run_id_dir!r}") from e - - def run_inner(self): - """ - Override this method or the run_with_llm() method. - """ - llm_executor: LLMExecutor = self.create_llm_executor() - - # Attempt executing this code with the first LLM, if that fails, try the next one, and so on. - def execute_function(llm: LLM) -> None: - self.run_with_llm(llm) - - # Make multiple attempts at running the run_with_llm() function. - # No try/except needed here. Let PlanTask.run() handle it. - llm_executor.run(execute_function) - - def run_with_llm(self, llm: LLM) -> None: - """ - Override this method or the run_inner() method. - """ - raise NotImplementedError("Subclasses must implement this method.") - - -class StartTimeTask(PlanTask): - """The timestamp when the pipeline was started.""" - def output(self): - return self.local_target(FilenameEnum.START_TIME) - - def run(self): - # The Gradio/Flask app that starts the luigi pipeline, must first create the `START_TIME` file inside the `run_id_dir`. - # This code will ONLY run if the Gradio/Flask app *failed* to create the file. - raise AssertionError(f"This code is not supposed to be run. Before starting the pipeline the '{FilenameEnum.START_TIME.value}' file must be present in the `run_id_dir`: {self.run_id_dir!r}") - - -class SetupTask(PlanTask): - """The plan prompt text provided by the user.""" - def output(self): - return self.local_target(FilenameEnum.INITIAL_PLAN) - - def run(self): - # The Gradio/Flask app that starts the luigi pipeline, must first create the `INITIAL_PLAN` file inside the `run_id_dir`. - # This code will ONLY run if the Gradio/Flask app *failed* to create the file. - raise AssertionError(f"This code is not supposed to be run. Before starting the pipeline the '{FilenameEnum.INITIAL_PLAN.value}' file must be present in the `run_id_dir`: {self.run_id_dir!r}") - - -class RedlineGateTask(PlanTask): - def requires(self): - return self.clone(SetupTask) - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.REDLINE_GATE_RAW), - 'markdown': self.local_target(FilenameEnum.REDLINE_GATE_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input().open("r") as f: - plan_prompt = f.read() - - redline_gate = RedlineGate.execute(llm, plan_prompt) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - redline_gate.save_raw(output_raw_path) - output_markdown_path = self.output()['markdown'].path - redline_gate.save_markdown(output_markdown_path) - - -class PremiseAttackTask(PlanTask): - def requires(self): - return self.clone(SetupTask) - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PREMISE_ATTACK_RAW), - 'markdown': self.local_target(FilenameEnum.PREMISE_ATTACK_MARKDOWN) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input().open("r") as f: - plan_prompt = f.read() - - premise_attack = PremiseAttack.execute(llm_executor, plan_prompt) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - premise_attack.save_raw(output_raw_path) - output_markdown_path = self.output()['markdown'].path - premise_attack.save_markdown(output_markdown_path) - - -class IdentifyPurposeTask(PlanTask): - """ - Determine if this is this going to be a business/personal/other plan. - """ - def requires(self): - return self.clone(SetupTask) - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.IDENTIFY_PURPOSE_RAW), - 'markdown': self.local_target(FilenameEnum.IDENTIFY_PURPOSE_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input().open("r") as f: - plan_prompt = f.read() - - identify_purpose = IdentifyPurpose.execute(llm, plan_prompt) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - identify_purpose.save_raw(output_raw_path) - output_markdown_path = self.output()['markdown'].path - identify_purpose.save_markdown(output_markdown_path) - - -class PlanTypeTask(PlanTask): - """ - Determine if the plan is purely digital or requires physical locations. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PLAN_TYPE_RAW), - 'markdown': self.local_target(FilenameEnum.PLAN_TYPE_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}" - ) - - identify_plan_type = IdentifyPlanType.execute(llm, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - identify_plan_type.save_raw(str(output_raw_path)) - output_markdown_path = self.output()['markdown'].path - identify_plan_type.save_markdown(str(output_markdown_path)) - -class PotentialLeversTask(PlanTask): - """ - Identify potential levers that can be adjusted. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.POTENTIAL_LEVERS_RAW), - 'clean': self.local_target(FilenameEnum.POTENTIAL_LEVERS_CLEAN), - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}" - ) - - identify_potential_levers = IdentifyPotentialLevers.execute(llm_executor, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - identify_potential_levers.save_raw(str(output_raw_path)) - output_clean_path = self.output()['clean'].path - identify_potential_levers.save_clean(str(output_clean_path)) - - -class DeduplicateLeversTask(PlanTask): - """ - The potential levers usually have some redundant levers. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'potential_levers': self.clone(PotentialLeversTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.DEDUPLICATED_LEVERS_RAW) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['potential_levers']['clean'].open("r") as f: - lever_item_list = json.load(f) - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}" - ) - - deduplicate_levers = DeduplicateLevers.execute( - llm_executor, - project_context=query, - raw_levers_list=lever_item_list - ) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - deduplicate_levers.save_raw(str(output_raw_path)) - -class EnrichLeversTask(PlanTask): - """ - Enrich potential levers with more information. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'deduplicate_levers': self.clone(DeduplicateLeversTask), - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.ENRICHED_LEVERS_RAW) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['deduplicate_levers']['raw'].open("r") as f: - json_dict = json.load(f) - lever_item_list = json_dict["deduplicated_levers"] - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}" - ) - - enrich_potential_levers = EnrichPotentialLevers.execute( - llm_executor, - project_context=query, - raw_levers_list=lever_item_list - ) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - enrich_potential_levers.save_raw(str(output_raw_path)) - -class FocusOnVitalFewLeversTask(PlanTask): - """ - Apply the 80/20 principle to the levers. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'enriched_levers': self.clone(EnrichLeversTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.VITAL_FEW_LEVERS_RAW) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['enriched_levers']['raw'].open("r") as f: - lever_item_list = json.load(f)["characterized_levers"] - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - ) - - focus_on_vital_few_levers = FocusOnVitalFewLevers.execute( - llm_executor, - project_context=query, - raw_levers_list=lever_item_list - ) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - focus_on_vital_few_levers.save_raw(str(output_raw_path)) - - -class StrategicDecisionsMarkdownTask(PlanTask): - """ - Human readable markdown with the levers. - """ - def requires(self): - return { - 'enriched_levers': self.clone(EnrichLeversTask), - 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask) - } - - def output(self): - return { - 'markdown': self.local_target(FilenameEnum.STRATEGIC_DECISIONS_MARKDOWN) - } - - def run(self): - with self.input()['enriched_levers']['raw'].open("r") as f: - enrich_lever_list = json.load(f)["characterized_levers"] - with self.input()['levers_vital_few']['raw'].open("r") as f: - vital_data = json.load(f) - vital_lever_list = vital_data["levers"] - lever_assessments_list = vital_data.get("response", {}).get("lever_assessments", []) - vital_levers_summary = vital_data.get("response", {}).get("summary", "") - - result = StrategicDecisionsMarkdown(enrich_lever_list, vital_lever_list, vital_levers_summary, lever_assessments_list) - result.save_markdown(self.output()['markdown'].path) - - -class CandidateScenariosTask(PlanTask): - """ - Combinations of the vital few levers. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.CANDIDATE_SCENARIOS_RAW), - 'clean': self.local_target(FilenameEnum.CANDIDATE_SCENARIOS_CLEAN) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['levers_vital_few']['raw'].open("r") as f: - lever_item_list = json.load(f)["levers"] - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - ) - - scenarios = CandidateScenarios.execute( - llm_executor=llm_executor, - project_context=query, - raw_vital_levers=lever_item_list - ) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - scenarios.save_raw(str(output_raw_path)) - output_clean_path = self.output()['clean'].path - scenarios.save_clean(str(output_clean_path)) - - -class SelectScenarioTask(PlanTask): - """ - Pick the best fitting scenario to make a plan for. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'levers_vital_few': self.clone(FocusOnVitalFewLeversTask), - 'candidate_scenarios': self.clone(CandidateScenariosTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.SELECTED_SCENARIO_RAW), - 'clean': self.local_target(FilenameEnum.SELECTED_SCENARIO_CLEAN) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['levers_vital_few']['raw'].open("r") as f: - lever_item_list = json.load(f)["levers"] - with self.input()['candidate_scenarios']['clean'].open("r") as f: - scenarios_list = json.load(f).get('scenarios', []) - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - f"File 'levers_vital_few.json':\n{format_json_for_use_in_query(lever_item_list)}\n\n" - f"File 'candidate_scenarios.json':\n{format_json_for_use_in_query(scenarios_list)}" - ) - - select_scenario = SelectScenario.execute( - llm_executor=llm_executor, - project_context=query, - scenarios=scenarios_list - ) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - select_scenario.save_raw(str(output_raw_path)) - output_clean_path = self.output()['clean'].path - select_scenario.save_clean(str(output_clean_path)) - - -class ScenariosMarkdownTask(PlanTask): - """ - Present the scenarios in a human readable format. - """ - def requires(self): - return { - 'candidate_scenarios': self.clone(CandidateScenariosTask), - 'selected_scenario': self.clone(SelectScenarioTask) - } - - def output(self): - return { - 'markdown': self.local_target(FilenameEnum.SCENARIOS_MARKDOWN) - } - - def run(self): - with self.input()['candidate_scenarios']['clean'].open("r") as f: - scenarios_list = json.load(f).get('scenarios', []) - with self.input()['selected_scenario']['clean'].open("r") as f: - selected_scenario_dict = json.load(f) - - # Extract the required data from the selected scenario - plan_characteristics = selected_scenario_dict.get('plan_characteristics', {}) - scenario_assessments = selected_scenario_dict.get('scenario_assessments', []) - final_choice = selected_scenario_dict.get('final_choice', {}) - - result = ScenariosMarkdown(scenarios_list, plan_characteristics, scenario_assessments, final_choice) - result.save_markdown(self.output()['markdown'].path) - - -class PhysicalLocationsTask(PlanTask): - """ - Identify/suggest physical locations for the plan. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PHYSICAL_LOCATIONS_RAW), - 'markdown': self.local_target(FilenameEnum.PHYSICAL_LOCATIONS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Identify/suggest physical locations for the plan...") - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['raw'].open("r") as f: - plan_type_dict = json.load(f) - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - - output_raw_path = self.output()['raw'].path - output_markdown_path = self.output()['markdown'].path - - plan_type = plan_type_dict.get("plan_type") - if plan_type == "physical": - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}" - ) - - physical_locations = PhysicalLocations.execute(llm, query) - - # Write the physical locations to disk. - physical_locations.save_raw(str(output_raw_path)) - physical_locations.save_markdown(str(output_markdown_path)) - else: - # Write an empty file to indicate that there are no physical locations. - data = { - "comment": "The plan is purely digital, without any physical locations." - } - with open(output_raw_path, "w") as f: - json.dump(data, f, indent=2) - - with open(output_markdown_path, "w", encoding='utf-8') as f: - f.write("The plan is purely digital, without any physical locations.") - -class CurrencyStrategyTask(PlanTask): - """ - Identify/suggest what currency to use for the plan, depending on the physical locations. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.CURRENCY_STRATEGY_RAW), - 'markdown': self.local_target(FilenameEnum.CURRENCY_STRATEGY_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['physical_locations']['markdown'].open("r") as f: - physical_locations_markdown = f.read() - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'physical_locations.md':\n{physical_locations_markdown}" - ) - - currency_strategy = CurrencyStrategy.execute(llm, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - currency_strategy.save_raw(str(output_raw_path)) - output_markdown_path = self.output()['markdown'].path - currency_strategy.save_markdown(str(output_markdown_path)) - - -class IdentifyRisksTask(PlanTask): - """ - Identify risks for the plan, depending on the physical locations. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'currency_strategy': self.clone(CurrencyStrategyTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.IDENTIFY_RISKS_RAW), - 'markdown': self.local_target(FilenameEnum.IDENTIFY_RISKS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['physical_locations']['markdown'].open("r") as f: - physical_locations_markdown = f.read() - with self.input()['currency_strategy']['markdown'].open("r") as f: - currency_strategy_markdown = f.read() - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'physical_locations.md':\n{physical_locations_markdown}\n\n" - f"File 'currency_strategy.md':\n{currency_strategy_markdown}" - ) - - identify_risks = IdentifyRisks.execute(llm, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - identify_risks.save_raw(str(output_raw_path)) - output_markdown_path = self.output()['markdown'].path - identify_risks.save_markdown(str(output_markdown_path)) - - -class MakeAssumptionsTask(PlanTask): - """ - Make assumptions about the plan. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'currency_strategy': self.clone(CurrencyStrategyTask), - 'identify_risks': self.clone(IdentifyRisksTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_RAW), - 'clean': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_CLEAN), - 'markdown': self.local_target(FilenameEnum.MAKE_ASSUMPTIONS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['plan_type']['markdown'].open("r") as f: - plan_type_markdown = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['physical_locations']['markdown'].open("r") as f: - physical_locations_markdown = f.read() - with self.input()['currency_strategy']['markdown'].open("r") as f: - currency_strategy_markdown = f.read() - with self.input()['identify_risks']['markdown'].open("r") as f: - identify_risks_markdown = f.read() - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'plan_type.md':\n{plan_type_markdown}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'physical_locations.md':\n{physical_locations_markdown}\n\n" - f"File 'currency_strategy.md':\n{currency_strategy_markdown}\n\n" - f"File 'identify_risks.md':\n{identify_risks_markdown}" - ) - - make_assumptions = MakeAssumptions.execute(llm, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - make_assumptions.save_raw(str(output_raw_path)) - output_clean_path = self.output()['clean'].path - make_assumptions.save_assumptions(str(output_clean_path)) - output_markdown_path = self.output()['markdown'].path - make_assumptions.save_markdown(str(output_markdown_path)) - - -class DistillAssumptionsTask(PlanTask): - """ - Distill raw assumption data. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'make_assumptions': self.clone(MakeAssumptionsTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.DISTILL_ASSUMPTIONS_RAW), - 'markdown': self.local_target(FilenameEnum.DISTILL_ASSUMPTIONS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['identify_purpose']['markdown'].open("r") as f: - identify_purpose_markdown = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - make_assumptions_target = self.input()['make_assumptions']['clean'] - with make_assumptions_target.open("r") as f: - assumptions_raw_data = json.load(f) - - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'purpose.md':\n{identify_purpose_markdown}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.json':\n{format_json_for_use_in_query(assumptions_raw_data)}" - ) - - distill_assumptions = DistillAssumptions.execute(llm, query) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - distill_assumptions.save_raw(str(output_raw_path)) - output_markdown_path = self.output()['markdown'].path - distill_assumptions.save_markdown(str(output_markdown_path)) - - -class ReviewAssumptionsTask(PlanTask): - """ - Find issues with the assumptions. - """ - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'currency_strategy': self.clone(CurrencyStrategyTask), - 'identify_risks': self.clone(IdentifyRisksTask), - 'make_assumptions': self.clone(MakeAssumptionsTask), - 'distill_assumptions': self.clone(DistillAssumptionsTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.REVIEW_ASSUMPTIONS_RAW), - 'markdown': self.local_target(FilenameEnum.REVIEW_ASSUMPTIONS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Define the list of (title, path) tuples - title_path_list = [ - ('Purpose', self.input()['identify_purpose']['markdown'].path), - ('Plan Type', self.input()['plan_type']['markdown'].path), - ('Strategic Decisions', self.input()['strategic_decisions_markdown']['markdown'].path), - ('Scenarios', self.input()['scenarios_markdown']['markdown'].path), - ('Physical Locations', self.input()['physical_locations']['markdown'].path), - ('Currency Strategy', self.input()['currency_strategy']['markdown'].path), - ('Identify Risks', self.input()['identify_risks']['markdown'].path), - ('Make Assumptions', self.input()['make_assumptions']['markdown'].path), - ('Distill Assumptions', self.input()['distill_assumptions']['markdown'].path) - ] - - # Read the files and handle exceptions - markdown_chunks = [] - for title, path in title_path_list: - try: - with open(path, 'r', encoding='utf-8') as f: - markdown_chunk = f.read() - markdown_chunks.append(f"# {title}\n\n{markdown_chunk}") - except FileNotFoundError: - logger.warning(f"Markdown file not found: {path} (from {title})") - markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") - except Exception as e: - logger.error(f"Error reading markdown file {path} (from {title}): {e}") - markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") - - # Combine the markdown chunks - full_markdown = "\n\n".join(markdown_chunks) - - review_assumptions = ReviewAssumptions.execute(llm, full_markdown) - - # Write the result to disk. - output_raw_path = self.output()['raw'].path - review_assumptions.save_raw(str(output_raw_path)) - output_markdown_path = self.output()['markdown'].path - review_assumptions.save_markdown(str(output_markdown_path)) - - -class ConsolidateAssumptionsMarkdownTask(PlanTask): - """ - Combines multiple small markdown documents into a single big document. - """ - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'currency_strategy': self.clone(CurrencyStrategyTask), - 'identify_risks': self.clone(IdentifyRisksTask), - 'make_assumptions': self.clone(MakeAssumptionsTask), - 'distill_assumptions': self.clone(DistillAssumptionsTask), - 'review_assumptions': self.clone(ReviewAssumptionsTask) - } - - def output(self): - return { - 'full': self.local_target(FilenameEnum.CONSOLIDATE_ASSUMPTIONS_FULL_MARKDOWN), - 'short': self.local_target(FilenameEnum.CONSOLIDATE_ASSUMPTIONS_SHORT_MARKDOWN) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Define the list of (title, path) tuples - title_path_list = [ - ('Purpose', self.input()['identify_purpose']['markdown'].path), - ('Plan Type', self.input()['plan_type']['markdown'].path), - ('Physical Locations', self.input()['physical_locations']['markdown'].path), - ('Currency Strategy', self.input()['currency_strategy']['markdown'].path), - ('Identify Risks', self.input()['identify_risks']['markdown'].path), - ('Make Assumptions', self.input()['make_assumptions']['markdown'].path), - ('Distill Assumptions', self.input()['distill_assumptions']['markdown'].path), - ('Review Assumptions', self.input()['review_assumptions']['markdown'].path) - ] - - # Read the files and handle exceptions - full_markdown_chunks = [] - short_markdown_chunks = [] - for title, path in title_path_list: - try: - with open(path, 'r', encoding='utf-8') as f: - markdown_chunk = f.read() - full_markdown_chunks.append(f"# {title}\n\n{markdown_chunk}") - except FileNotFoundError: - logger.warning(f"Markdown file not found: {path} (from {title})") - full_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") - short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nFile not found.") - continue - except Exception as e: - logger.error(f"Error reading markdown file {path} (from {title}): {e}") - full_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") - short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError reading markdown file.") - continue - - # IDEA: If the chunk file already exist, then there is no need to run the LLM again. - def execute_shorten_markdown(llm: LLM) -> ShortenMarkdown: - return ShortenMarkdown.execute(llm, markdown_chunk) - - try: - shorten_markdown = llm_executor.run(execute_shorten_markdown) - short_markdown_chunks.append(f"# {title}\n{shorten_markdown.markdown}") - except PipelineStopRequested: - # Re-raise PipelineStopRequested without wrapping it - raise - except Exception as e: - logger.error(f"Error shortening markdown file {path} (from {title}): {e}") - short_markdown_chunks.append(f"**Problem with document:** '{title}'\n\nError shortening markdown file.") - continue - - # Combine the markdown chunks - full_markdown = "\n\n".join(full_markdown_chunks) - short_markdown = "\n\n".join(short_markdown_chunks) - - # Write the result to disk. - output_full_markdown_path = self.output()['full'].path - with open(output_full_markdown_path, "w", encoding="utf-8") as f: - f.write(full_markdown) - - output_short_markdown_path = self.output()['short'].path - with open(output_short_markdown_path, "w", encoding="utf-8") as f: - f.write(short_markdown) - - -class PreProjectAssessmentTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PRE_PROJECT_ASSESSMENT_RAW), - 'clean': self.local_target(FilenameEnum.PRE_PROJECT_ASSESSMENT) - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Conducting pre-project assessment...") - - # Read the plan prompt from the SetupTask's output. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - - # Build the query. - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}" - ) - - # Execute the pre-project assessment. - pre_project_assessment = PreProjectAssessment.execute(llm, query) - - # Save raw output. - raw_path = self.file_path(FilenameEnum.PRE_PROJECT_ASSESSMENT_RAW) - pre_project_assessment.save_raw(str(raw_path)) - - # Save cleaned pre-project assessment. - clean_path = self.file_path(FilenameEnum.PRE_PROJECT_ASSESSMENT) - pre_project_assessment.save_preproject_assessment(str(clean_path)) - - -class ProjectPlanTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PROJECT_PLAN_RAW), - 'markdown': self.local_target(FilenameEnum.PROJECT_PLAN_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Creating plan...") - - # Read the plan prompt from SetupTask's output. - setup_target = self.input()['setup'] - with setup_target.open("r") as f: - plan_prompt = f.read() - - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - - # Load the consolidated assumptions. - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - - # Read the pre-project assessment from its file. - pre_project_assessment_file = self.input()['preproject']['clean'] - with pre_project_assessment_file.open("r") as f: - pre_project_assessment_dict = json.load(f) - - # Build the query. - query = ( - f"File 'plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}" - ) - - # Execute the plan creation. - project_plan = ProjectPlan.execute(llm, query) - - # Save raw output - project_plan.save_raw(self.output()['raw'].path) - - # Save markdown output - project_plan.save_markdown(self.output()['markdown'].path) - - logger.info("Project plan created and saved") - - -class GovernancePhase1AuditTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE1_AUDIT_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE1_AUDIT_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}" - ) - - # Execute. - try: - governance_phase1_audit = GovernancePhase1Audit.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase1Audit failed: %s", e) - raise - - # Save the results. - governance_phase1_audit.save_raw(self.output()['raw'].path) - governance_phase1_audit.save_markdown(self.output()['markdown'].path) - - -class GovernancePhase2BodiesTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE2_BODIES_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE2_BODIES_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['governance_phase1_audit']['markdown'].open("r") as f: - governance_phase1_audit_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'governance-phase1-audit.md':\n{governance_phase1_audit_markdown}" - ) - - # Execute. - try: - governance_phase2_bodies = GovernancePhase2Bodies.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase2Bodies failed: %s", e) - raise - - # Save the results. - governance_phase2_bodies.save_raw(self.output()['raw'].path) - governance_phase2_bodies.save_markdown(self.output()['markdown'].path) - - -class GovernancePhase3ImplPlanTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE3_IMPL_PLAN_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE3_IMPL_PLAN_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - with self.input()['governance_phase2_bodies']['raw'].open("r") as f: - governance_phase2_bodies_dict = json.load(f) - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" - f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}" - ) - - # Execute. - try: - governance_phase3_impl_plan = GovernancePhase3ImplPlan.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase3ImplPlan failed: %s", e) - raise - - # Save the results. - governance_phase3_impl_plan.save_raw(self.output()['raw'].path) - governance_phase3_impl_plan.save_markdown(self.output()['markdown'].path) - -class GovernancePhase4DecisionEscalationMatrixTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), - 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE4_DECISION_ESCALATION_MATRIX_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE4_DECISION_ESCALATION_MATRIX_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - with self.input()['governance_phase2_bodies']['raw'].open("r") as f: - governance_phase2_bodies_dict = json.load(f) - with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: - governance_phase3_impl_plan_dict = json.load(f) - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" - f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" - f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}" - ) - - # Execute. - try: - governance_phase4_decision_escalation_matrix = GovernancePhase4DecisionEscalationMatrix.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase4DecisionEscalationMatrix failed: %s", e) - raise - - # Save the results. - governance_phase4_decision_escalation_matrix.save_raw(self.output()['raw'].path) - governance_phase4_decision_escalation_matrix.save_markdown(self.output()['markdown'].path) - -class GovernancePhase5MonitoringProgressTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), - 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), - 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE5_MONITORING_PROGRESS_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE5_MONITORING_PROGRESS_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - with self.input()['governance_phase2_bodies']['raw'].open("r") as f: - governance_phase2_bodies_dict = json.load(f) - with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: - governance_phase3_impl_plan_dict = json.load(f) - with self.input()['governance_phase4_decision_escalation_matrix']['raw'].open("r") as f: - governance_phase4_decision_escalation_matrix_dict = json.load(f) - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" - f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" - f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}\n\n" - f"File 'governance-phase4-decision-escalation-matrix.json':\n{format_json_for_use_in_query(governance_phase4_decision_escalation_matrix_dict)}" - ) - - # Execute. - try: - governance_phase5_monitoring_progress = GovernancePhase5MonitoringProgress.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase5MonitoringProgress failed: %s", e) - raise - - # Save the results. - governance_phase5_monitoring_progress.save_raw(self.output()['raw'].path) - governance_phase5_monitoring_progress.save_markdown(self.output()['markdown'].path) - -class GovernancePhase6ExtraTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), - 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), - 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), - 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.GOVERNANCE_PHASE6_EXTRA_RAW), - 'markdown': self.local_target(FilenameEnum.GOVERNANCE_PHASE6_EXTRA_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - with self.input()['governance_phase1_audit']['raw'].open("r") as f: - governance_phase1_audit_dict = json.load(f) - with self.input()['governance_phase2_bodies']['raw'].open("r") as f: - governance_phase2_bodies_dict = json.load(f) - with self.input()['governance_phase3_impl_plan']['raw'].open("r") as f: - governance_phase3_impl_plan_dict = json.load(f) - with self.input()['governance_phase4_decision_escalation_matrix']['raw'].open("r") as f: - governance_phase4_decision_escalation_matrix_dict = json.load(f) - with self.input()['governance_phase5_monitoring_progress']['raw'].open("r") as f: - governance_phase5_monitoring_progress_dict = json.load(f) - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}\n\n" - f"File 'governance-phase1-audit.json':\n{format_json_for_use_in_query(governance_phase1_audit_dict)}\n\n" - f"File 'governance-phase2-bodies.json':\n{format_json_for_use_in_query(governance_phase2_bodies_dict)}\n\n" - f"File 'governance-phase3-impl-plan.json':\n{format_json_for_use_in_query(governance_phase3_impl_plan_dict)}\n\n" - f"File 'governance-phase4-decision-escalation-matrix.json':\n{format_json_for_use_in_query(governance_phase4_decision_escalation_matrix_dict)}\n\n" - f"File 'governance-phase5-monitoring-progress.json':\n{format_json_for_use_in_query(governance_phase5_monitoring_progress_dict)}" - ) - - # Execute. - try: - governance_phase6_extra = GovernancePhase6Extra.execute(llm, query) - except Exception as e: - logger.error("GovernancePhase6Extra failed: %s", e) - raise - - # Save the results. - governance_phase6_extra.save_raw(self.output()['raw'].path) - governance_phase6_extra.save_markdown(self.output()['markdown'].path) - -class ConsolidateGovernanceTask(PlanTask): - def requires(self): - return { - 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), - 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), - 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), - 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask), - 'governance_phase6_extra': self.clone(GovernancePhase6ExtraTask) - } - - def output(self): - return self.local_target(FilenameEnum.CONSOLIDATE_GOVERNANCE_MARKDOWN) - - def run_inner(self): - # Read inputs from required tasks. - with self.input()['governance_phase1_audit']['markdown'].open("r") as f: - governance_phase1_audit_markdown = f.read() - with self.input()['governance_phase2_bodies']['markdown'].open("r") as f: - governance_phase2_bodies_markdown = f.read() - with self.input()['governance_phase3_impl_plan']['markdown'].open("r") as f: - governance_phase3_impl_plan_markdown = f.read() - with self.input()['governance_phase4_decision_escalation_matrix']['markdown'].open("r") as f: - governance_phase4_decision_escalation_matrix_markdown = f.read() - with self.input()['governance_phase5_monitoring_progress']['markdown'].open("r") as f: - governance_phase5_monitoring_progress_markdown = f.read() - with self.input()['governance_phase6_extra']['markdown'].open("r") as f: - governance_phase6_extra_markdown = f.read() - - # Build the document. - markdown = [] - markdown.append(f"# Governance Audit\n\n{governance_phase1_audit_markdown}") - markdown.append(f"# Internal Governance Bodies\n\n{governance_phase2_bodies_markdown}") - markdown.append(f"# Governance Implementation Plan\n\n{governance_phase3_impl_plan_markdown}") - markdown.append(f"# Decision Escalation Matrix\n\n{governance_phase4_decision_escalation_matrix_markdown}") - markdown.append(f"# Monitoring Progress\n\n{governance_phase5_monitoring_progress_markdown}") - markdown.append(f"# Governance Extra\n\n{governance_phase6_extra_markdown}") - - content = "\n\n".join(markdown) - - with self.output().open("w") as f: - f.write(content) - -class RelatedResourcesTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.RELATED_RESOURCES_RAW), - 'markdown': self.local_target(FilenameEnum.RELATED_RESOURCES_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'project-plan.json':\n{format_json_for_use_in_query(project_plan_dict)}" - ) - - # Execute. - try: - related_resources = RelatedResources.execute(llm, query) - except Exception as e: - logger.error("SimilarProjects failed: %s", e) - raise - - # Save the results. - related_resources.save_raw(self.output()['raw'].path) - related_resources.save_markdown(self.output()['markdown'].path) - -class FindTeamMembersTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'related_resources': self.clone(RelatedResourcesTask), - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.FIND_TEAM_MEMBERS_RAW), - 'clean': self.local_target(FilenameEnum.FIND_TEAM_MEMBERS_CLEAN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute. - try: - find_team_members = FindTeamMembers.execute(llm, query) - except Exception as e: - logger.error("FindTeamMembers failed: %s", e) - raise - - # Save the raw output. - raw_dict = find_team_members.to_dict() - with self.output()['raw'].open("w") as f: - json.dump(raw_dict, f, indent=2) - - # Save the cleaned up result. - team_member_list = find_team_members.team_member_list - with self.output()['clean'].open("w") as f: - json.dump(team_member_list, f, indent=2) - -class EnrichTeamMembersWithContractTypeTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'find_team_members': self.clone(FindTeamMembersTask), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_CONTRACT_TYPE_RAW), - 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_CONTRACT_TYPE_CLEAN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['find_team_members']['clean'].open("r") as f: - team_member_list = json.load(f) - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute. - try: - enrich_team_members_with_contract_type = EnrichTeamMembersWithContractType.execute(llm, query, team_member_list) - except Exception as e: - logger.error("EnrichTeamMembersWithContractType failed: %s", e) - raise - - # Save the raw output. - raw_dict = enrich_team_members_with_contract_type.to_dict() - with self.output()['raw'].open("w") as f: - json.dump(raw_dict, f, indent=2) - - # Save the cleaned up result. - team_member_list = enrich_team_members_with_contract_type.team_member_list - with self.output()['clean'].open("w") as f: - json.dump(team_member_list, f, indent=2) - -class EnrichTeamMembersWithBackgroundStoryTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'enrich_team_members_with_contract_type': self.clone(EnrichTeamMembersWithContractTypeTask), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_BACKGROUND_STORY_RAW), - 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_BACKGROUND_STORY_CLEAN) - } - - def run_with_llm(self, llm: LLM) -> None: - # In FAST_BUT_SKIP_DETAILS mode, skip fictional biography generation entirely. - # Pass the bare team member list (title, domain, relevance) through unchanged. - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode: skipping biography enrichment.") - with self.input()['enrich_team_members_with_contract_type']['clean'].open("r") as f: - team_member_list = json.load(f) - with self.output()['raw'].open("w") as f: - json.dump({"team_members": team_member_list}, f, indent=2) - with self.output()['clean'].open("w") as f: - json.dump(team_member_list, f, indent=2) - return - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['enrich_team_members_with_contract_type']['clean'].open("r") as f: - team_member_list = json.load(f) - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute. - try: - enrich_team_members_with_background_story = EnrichTeamMembersWithBackgroundStory.execute(llm, query, team_member_list) - except Exception as e: - logger.error("EnrichTeamMembersWithBackgroundStory failed: %s", e) - raise - - # Save the raw output. - raw_dict = enrich_team_members_with_background_story.to_dict() - with self.output()['raw'].open("w") as f: - json.dump(raw_dict, f, indent=2) - - # Save the cleaned up result. - team_member_list = enrich_team_members_with_background_story.team_member_list - with self.output()['clean'].open("w") as f: - json.dump(team_member_list, f, indent=2) - -class EnrichTeamMembersWithEnvironmentInfoTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'enrich_team_members_with_background_story': self.clone(EnrichTeamMembersWithBackgroundStoryTask), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_ENVIRONMENT_INFO_RAW), - 'clean': self.local_target(FilenameEnum.ENRICH_TEAM_MEMBERS_ENVIRONMENT_INFO_CLEAN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['enrich_team_members_with_background_story']['clean'].open("r") as f: - team_member_list = json.load(f) - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'team-members-that-needs-to-be-enriched.json':\n{format_json_for_use_in_query(team_member_list)}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute. - try: - enrich_team_members_with_background_story = EnrichTeamMembersWithEnvironmentInfo.execute(llm, query, team_member_list) - except Exception as e: - logger.error("EnrichTeamMembersWithEnvironmentInfo failed: %s", e) - raise - - # Save the raw output. - raw_dict = enrich_team_members_with_background_story.to_dict() - with self.output()['raw'].open("w") as f: - json.dump(raw_dict, f, indent=2) - - # Save the cleaned up result. - team_member_list = enrich_team_members_with_background_story.team_member_list - with self.output()['clean'].open("w") as f: - json.dump(team_member_list, f, indent=2) - -class ReviewTeamTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def output(self): - return self.local_target(FilenameEnum.REVIEW_TEAM_RAW) - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['enrich_team_members_with_environment_info']['clean'].open("r") as f: - team_member_list = json.load(f) - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Convert the team members to a Markdown document. - builder = TeamMarkdownDocumentBuilder() - builder.append_roles(team_member_list, title=None) - team_document_markdown = builder.to_string() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'team-members.md':\n{team_document_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute. - try: - review_team = ReviewTeam.execute(llm, query) - except Exception as e: - logger.error("ReviewTeam failed: %s", e) - raise - - # Save the raw output. - raw_dict = review_team.to_dict() - with self.output().open("w") as f: - json.dump(raw_dict, f, indent=2) - - logger.info("ReviewTeamTask complete.") - -class TeamMarkdownTask(PlanTask): - def requires(self): - return { - 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), - 'review_team': self.clone(ReviewTeamTask) - } - - def output(self): - return self.local_target(FilenameEnum.TEAM_MARKDOWN) - - def run_inner(self): - logger.info("TeamMarkdownTask. Loading files...") - - # 1. Read the team_member_list from EnrichTeamMembersWithEnvironmentInfoTask. - with self.input()['enrich_team_members_with_environment_info']['clean'].open("r") as f: - team_member_list = json.load(f) - - # 2. Read the json from ReviewTeamTask. - with self.input()['review_team'].open("r") as f: - review_team_json = json.load(f) - - logger.info("TeamMarkdownTask. All files are now ready. Processing...") - - # Combine the team members and the review into a Markdown document. - builder = TeamMarkdownDocumentBuilder() - builder.append_team_member_subtitle() - builder.append_roles(team_member_list) - builder.append_separator() - builder.append_full_review(review_team_json) - builder.write_to_file(self.output().path) - - logger.info("TeamMarkdownTask complete.") - -class SWOTAnalysisTask(PlanTask): - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.SWOT_RAW), - 'markdown': self.local_target(FilenameEnum.SWOT_MARKDOWN) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - consolidate_assumptions_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query for SWOT analysis. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{consolidate_assumptions_markdown}\n\n" - f"File 'pre-project-assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}" - ) - - # Execute the SWOT analysis. - # Send the identify_purpose_dict to SWOTAnalysis, and use business/personal/other to select the system prompt - try: - swot_analysis = SWOTAnalysis.execute(llm=llm, query=query, identify_purpose_dict=identify_purpose_dict) - except Exception as e: - logger.error("SWOT analysis failed: %s", e) - raise - - # Convert the SWOT analysis to a dict and markdown. - swot_raw_dict = swot_analysis.to_dict() - swot_markdown = swot_analysis.to_markdown(include_metadata=False, include_purpose=False) - - # Write the raw SWOT JSON. - with self.output()['raw'].open("w") as f: - json.dump(swot_raw_dict, f, indent=2) - - # Write the SWOT analysis as Markdown. - markdown_path = self.output()['markdown'].path - with open(markdown_path, "w", encoding="utf-8") as f: - f.write(swot_markdown) - -class ExpertReviewTask(PlanTask): - """ - Finds experts to review the SWOT analysis and have them provide criticism. - """ - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'preproject': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'swot_analysis': self.clone(SWOTAnalysisTask) - } - - def output(self): - return self.local_target(FilenameEnum.EXPERT_CRITICISM_MARKDOWN) - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - logger.info("Finding experts to review the SWOT analysis, and having them provide criticism...") - - # Read inputs from required tasks. - with self.input()['setup'].open("r") as f: - plan_prompt = f.read() - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['preproject']['clean'].open("r") as f: - pre_project_assessment_dict = json.load(f) - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - swot_markdown_path = self.input()['swot_analysis']['markdown'].path - with open(swot_markdown_path, "r", encoding="utf-8") as f: - swot_markdown = f.read() - - # Build the query. - query = ( - f"File 'initial-plan.txt':\n{plan_prompt}\n\n" - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'pre-project assessment.json':\n{format_json_for_use_in_query(pre_project_assessment_dict)}\n\n" - f"File 'project_plan.md':\n{project_plan_markdown}\n\n" - f"File 'SWOT Analysis.md':\n{swot_markdown}" - ) - - # Define callback functions. - def phase1_post_callback(expert_finder: ExpertFinder) -> None: - raw_path = self.run_id_dir / FilenameEnum.EXPERTS_RAW.value - cleaned_path = self.run_id_dir / FilenameEnum.EXPERTS_CLEAN.value - expert_finder.save_raw(str(raw_path)) - expert_finder.save_cleanedup(str(cleaned_path)) - - def phase2_post_callback(expert_criticism: ExpertCriticism, expert_index: int) -> None: - file_path = self.run_id_dir / FilenameEnum.EXPERT_CRITICISM_RAW_TEMPLATE.format(expert_index + 1) - expert_criticism.save_raw(str(file_path)) - - # Execute the expert orchestration. - expert_orchestrator = ExpertOrchestrator() - # IDEA: max_expert_count. don't truncate to 2 experts. Interview them all in production mode. - # IDEA: If the expert file for expert_index already exist, then there is no need to run the LLM again. - expert_orchestrator.phase1_post_callback = phase1_post_callback - expert_orchestrator.phase2_post_callback = phase2_post_callback - expert_orchestrator.execute(llm_executor, query) - - # Write final expert criticism markdown. - expert_criticism_markdown_file = self.file_path(FilenameEnum.EXPERT_CRITICISM_MARKDOWN) - with expert_criticism_markdown_file.open("w") as f: - f.write(expert_orchestrator.to_markdown()) - - -class DataCollectionTask(PlanTask): - """ - Determine what kind of data is to be collected. - """ - def output(self): - return { - 'raw': self.local_target(FilenameEnum.DATA_COLLECTION_RAW), - 'markdown': self.local_target(FilenameEnum.DATA_COLLECTION_MARKDOWN) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'expert_review': self.clone(ExpertReviewTask) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}" - ) - - # Invoke the LLM. - data_collection = DataCollection.execute(llm, query) - - # Save the results. - json_path = self.output()['raw'].path - data_collection.save_raw(json_path) - markdown_path = self.output()['markdown'].path - data_collection.save_markdown(markdown_path) - -class IdentifyDocumentsTask(PlanTask): - """ - Identify documents that need to be created or found for the project. - """ - def output(self): - return { - "raw": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_RAW), - "markdown": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_MARKDOWN), - "documents_to_find": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_TO_FIND_JSON), - "documents_to_create": self.local_target(FilenameEnum.IDENTIFIED_DOCUMENTS_TO_CREATE_JSON), - } - - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'expert_review': self.clone(ExpertReviewTask) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}" - ) - - # Invoke the LLM. - identify_documents = IdentifyDocuments.execute( - llm=llm, - user_prompt=query, - identify_purpose_dict=identify_purpose_dict - ) - - # Save the results. - identify_documents.save_raw(self.output()["raw"].path) - identify_documents.save_markdown(self.output()["markdown"].path) - identify_documents.save_json_documents_to_find(self.output()["documents_to_find"].path) - identify_documents.save_json_documents_to_create(self.output()["documents_to_create"].path) - -class FilterDocumentsToFindTask(PlanTask): - """ - The "documents to find" may be a long list of documents, some duplicates, irrelevant, not needed at an early stage of the project. - This task narrows down to a handful of relevant documents. - """ - def output(self): - return { - "raw": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_FIND_RAW), - "clean": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_FIND_CLEAN) - } - - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'identified_documents': self.clone(IdentifyDocumentsTask), - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['identified_documents']['documents_to_find'].open("r") as f: - documents_to_find = json.load(f) - - # Build the query. - process_documents, integer_id_to_document_uuid = FilterDocumentsToFind.process_documents_and_integer_ids(documents_to_find) - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'documents.json':\n{process_documents}" - ) - - # Invoke the LLM. - filter_documents = FilterDocumentsToFind.execute( - llm=llm, - user_prompt=query, - identified_documents_raw_json=documents_to_find, - integer_id_to_document_uuid=integer_id_to_document_uuid, - identify_purpose_dict=identify_purpose_dict - ) - - # Save the results. - filter_documents.save_raw(self.output()["raw"].path) - filter_documents.save_filtered_documents(self.output()["clean"].path) - -class FilterDocumentsToCreateTask(PlanTask): - """ - The "documents to create" may be a long list of documents, some duplicates, irrelevant, not needed at an early stage of the project. - This task narrows down to a handful of relevant documents. - """ - def output(self): - return { - "raw": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_CREATE_RAW), - "clean": self.local_target(FilenameEnum.FILTER_DOCUMENTS_TO_CREATE_CLEAN) - } - - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'identified_documents': self.clone(IdentifyDocumentsTask), - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['identified_documents']['documents_to_create'].open("r") as f: - documents_to_create = json.load(f) - - # Build the query. - process_documents, integer_id_to_document_uuid = FilterDocumentsToCreate.process_documents_and_integer_ids(documents_to_create) - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'documents.json':\n{process_documents}" - ) - - # Invoke the LLM. - filter_documents = FilterDocumentsToCreate.execute( - llm=llm, - user_prompt=query, - identified_documents_raw_json=documents_to_create, - integer_id_to_document_uuid=integer_id_to_document_uuid, - identify_purpose_dict=identify_purpose_dict - ) - - # Save the results. - filter_documents.save_raw(self.output()["raw"].path) - filter_documents.save_filtered_documents(self.output()["clean"].path) - -class DraftDocumentsToFindTask(PlanTask): - """ - The "documents to find". Write bullet points to what each document roughly should contain. - """ - def output(self): - return self.local_target(FilenameEnum.DRAFT_DOCUMENTS_TO_FIND_CONSOLIDATED) - - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'filter_documents_to_find': self.clone(FilterDocumentsToFindTask), - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['filter_documents_to_find']['clean'].open("r") as f: - documents_to_find = json.load(f) - - accumulated_documents = documents_to_find.copy() - - logger.info(f"DraftDocumentsToFindTask.speedvsdetail: {self.speedvsdetail}") - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") - documents_to_find = documents_to_find[:2] - else: - logger.info("Processing all chunks.") - - for index, document in enumerate(documents_to_find): - logger.info(f"Document-to-find: Drafting document {index+1} of {len(documents_to_find)}...") - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'document.json':\n{document}" - ) - - # IDEA: If the document already exist, then there is no need to run the LLM again. - def execute_draft_document_to_find(llm: LLM) -> DraftDocumentToFind: - return DraftDocumentToFind.execute(llm=llm, user_prompt=query, identify_purpose_dict=identify_purpose_dict) - - try: - draft_document = llm_executor.run(execute_draft_document_to_find) - except PipelineStopRequested: - # Re-raise PipelineStopRequested without wrapping it - raise - except Exception as e: - logger.error(f"Document-to-find {index+1} LLM interaction failed.", exc_info=True) - raise ValueError(f"Document-to-find {index+1} LLM interaction failed.") from e - - json_response = draft_document.to_dict() - - # Write the raw JSON for this document using the FilenameEnum template. - raw_filename = FilenameEnum.DRAFT_DOCUMENTS_TO_FIND_RAW_TEMPLATE.value.format(index+1) - raw_chunk_path = self.run_id_dir / raw_filename - with open(raw_chunk_path, 'w') as f: - json.dump(json_response, f, indent=2) - - # Merge the draft document into the original document. - document_updated = document.copy() - for key in draft_document.response.keys(): - document_updated[key] = draft_document.response[key] - accumulated_documents[index] = document_updated - - # Write the accumulated documents to the output file. - with self.output().open("w") as f: - json.dump(accumulated_documents, f, indent=2) - -class DraftDocumentsToCreateTask(PlanTask): - """ - The "documents to create". Write bullet points to what each document roughly should contain. - """ - def output(self): - return self.local_target(FilenameEnum.DRAFT_DOCUMENTS_TO_CREATE_CONSOLIDATED) - - def requires(self): - return { - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'filter_documents_to_create': self.clone(FilterDocumentsToCreateTask), - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['identify_purpose']['raw'].open("r") as f: - identify_purpose_dict = json.load(f) - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['filter_documents_to_create']['clean'].open("r") as f: - documents_to_create = json.load(f) - - accumulated_documents = documents_to_create.copy() - - logger.info(f"DraftDocumentsToCreateTask.speedvsdetail: {self.speedvsdetail}") - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") - documents_to_create = documents_to_create[:2] - else: - logger.info("Processing all chunks.") - - for index, document in enumerate(documents_to_create): - logger.info(f"Document-to-create: Drafting document {index+1} of {len(documents_to_create)}...") - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'document.json':\n{document}" - ) - - # IDEA: If the document already exist, then there is no need to run the LLM again. - def execute_draft_document_to_create(llm: LLM) -> DraftDocumentToCreate: - return DraftDocumentToCreate.execute(llm=llm, user_prompt=query, identify_purpose_dict=identify_purpose_dict) - - try: - draft_document = llm_executor.run(execute_draft_document_to_create) - except PipelineStopRequested: - # Re-raise PipelineStopRequested without wrapping it - raise - except Exception as e: - logger.error(f"Document-to-create {index+1} LLM interaction failed.", exc_info=True) - raise ValueError(f"Document-to-create {index+1} LLM interaction failed.") from e - - json_response = draft_document.to_dict() - - # Write the raw JSON for this document using the FilenameEnum template. - raw_filename = FilenameEnum.DRAFT_DOCUMENTS_TO_CREATE_RAW_TEMPLATE.value.format(index+1) - raw_chunk_path = self.run_id_dir / raw_filename - with open(raw_chunk_path, 'w') as f: - json.dump(json_response, f, indent=2) - - # Merge the draft document into the original document. - document_updated = document.copy() - for key in draft_document.response.keys(): - document_updated[key] = draft_document.response[key] - accumulated_documents[index] = document_updated - - # Write the accumulated documents to the output file. - with self.output().open("w") as f: - json.dump(accumulated_documents, f, indent=2) - -class MarkdownWithDocumentsToCreateAndFindTask(PlanTask): - """ - Create markdown with the "documents to create and find" - """ - def output(self): - return self.local_target(FilenameEnum.DOCUMENTS_TO_CREATE_AND_FIND_MARKDOWN) - - def requires(self): - return { - 'draft_documents_to_create': self.clone(DraftDocumentsToCreateTask), - 'draft_documents_to_find': self.clone(DraftDocumentsToFindTask), - } - - def run_inner(self): - # Read inputs from required tasks. - with self.input()['draft_documents_to_create'].open("r") as f: - documents_to_create = json.load(f) - with self.input()['draft_documents_to_find'].open("r") as f: - documents_to_find = json.load(f) - - accumulated_rows = [] - accumulated_rows.append("# Documents to Create") - for index, document in enumerate(documents_to_create, start=1): - rows = markdown_rows_with_document_to_create(index, document) - accumulated_rows.extend(rows) - - accumulated_rows.append("\n\n# Documents to Find") - for index, document in enumerate(documents_to_find, start=1): - rows = markdown_rows_with_document_to_find(index, document) - accumulated_rows.extend(rows) - - markdown_representation = "\n".join(accumulated_rows) - - # Write the markdown to the output file. - output_file_path = self.output().path - with open(output_file_path, 'w', encoding='utf-8') as f: - f.write(markdown_representation) - -class CreateWBSLevel1Task(PlanTask): - """ - Creates the Work Breakdown Structure (WBS) Level 1. - Depends on: - - ProjectPlanTask: provides the project plan as JSON. - Produces: - - Raw WBS Level 1 output file (xxx-wbs_level1_raw.json) - - Cleaned up WBS Level 1 file (xxx-wbs_level1.json) - """ - def requires(self): - return { - 'project_plan': self.clone(ProjectPlanTask) - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.WBS_LEVEL1_RAW), - 'clean': self.local_target(FilenameEnum.WBS_LEVEL1), - 'project_title': self.local_target(FilenameEnum.WBS_LEVEL1_PROJECT_TITLE) - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Creating Work Breakdown Structure (WBS) Level 1...") - - # Read the project plan JSON from the dependency. - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - - # Build the query using the project plan. - query = format_json_for_use_in_query(project_plan_dict) - - # Execute the WBS Level 1 creation. - create_wbs_level1 = CreateWBSLevel1.execute(llm, query) - - # Save the raw output. - wbs_level1_raw_dict = create_wbs_level1.raw_response_dict() - with self.output()['raw'].open("w") as f: - json.dump(wbs_level1_raw_dict, f, indent=2) - - # Save the cleaned up result. - wbs_level1_result_json = create_wbs_level1.cleanedup_dict() - with self.output()['clean'].open("w") as f: - json.dump(wbs_level1_result_json, f, indent=2) - - # Save the project title. - with self.output()['project_title'].open("w") as f: - f.write(create_wbs_level1.project_title) - - logger.info("WBS Level 1 created successfully.") - -class CreateWBSLevel2Task(PlanTask): - """ - Creates the Work Breakdown Structure (WBS) Level 2. - Depends on: - - ProjectPlanTask: provides the project plan as JSON. - - CreateWBSLevel1Task: provides the cleaned WBS Level 1 result. - Produces: - - Raw WBS Level 2 output (007-wbs_level2_raw.json) - - Cleaned WBS Level 2 output (008-wbs_level2.json) - """ - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'wbs_level1': self.clone(CreateWBSLevel1Task), - 'data_collection': self.clone(DataCollectionTask), - } - - def output(self): - return { - 'raw': self.local_target(FilenameEnum.WBS_LEVEL2_RAW), - 'clean': self.local_target(FilenameEnum.WBS_LEVEL2) - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Creating Work Breakdown Structure (WBS) Level 2...") - - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['wbs_level1']['clean'].open("r") as f: - wbs_level1_result_json = json.load(f) - - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'project_plan.md':\n{project_plan_markdown}\n\n" - f"File 'WBS Level 1.json':\n{format_json_for_use_in_query(wbs_level1_result_json)}\n\n" - f"File 'data_collection.md':\n{data_collection_markdown}" - ) - - # Execute the WBS Level 2 creation. - create_wbs_level2 = CreateWBSLevel2.execute(llm, query) - - # Retrieve and write the raw output. - wbs_level2_raw_dict = create_wbs_level2.raw_response_dict() - with self.output()['raw'].open("w") as f: - json.dump(wbs_level2_raw_dict, f, indent=2) - - # Retrieve and write the cleaned output (e.g. major phases with subtasks). - with self.output()['clean'].open("w") as f: - json.dump(create_wbs_level2.major_phases_with_subtasks, f, indent=2) - - logger.info("WBS Level 2 created successfully.") - -class WBSProjectLevel1AndLevel2Task(PlanTask): - """ - Create a WBS project from the WBS Level 1 and Level 2 JSON files. - - It depends on: - - CreateWBSLevel1Task: providing the cleaned WBS Level 1 JSON. - - CreateWBSLevel2Task: providing the major phases with subtasks and the task UUIDs. - """ - def output(self): - return self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2) - - def requires(self): - return { - 'wbs_level1': self.clone(CreateWBSLevel1Task), - 'wbs_level2': self.clone(CreateWBSLevel2Task), - } - - def run_inner(self): - wbs_level1_path = self.input()['wbs_level1']['clean'].path - wbs_level2_path = self.input()['wbs_level2']['clean'].path - wbs_project = WBSPopulate.project_from_level1_json(wbs_level1_path) - WBSPopulate.extend_project_with_level2_json(wbs_project, wbs_level2_path) - - json_representation = json.dumps(wbs_project.to_dict(), indent=2) - with self.output().open("w") as f: - f.write(json_representation) - -class CreatePitchTask(PlanTask): - """ - Create a the pitch that explains the project plan, from multiple perspectives. - - This task depends on: - - ProjectPlanTask: provides the project plan JSON. - - WBSProjectLevel1AndLevel2Task: containing the top level of the project plan. - - The resulting pitch JSON is written to the file specified by FilenameEnum.PITCH. - """ - def output(self): - return self.local_target(FilenameEnum.PITCH_RAW) - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), - 'related_resources': self.clone(RelatedResourcesTask) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read the project plan JSON. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - - with self.input()['wbs_project'].open("r") as f: - wbs_project_dict = json.load(f) - wbs_project = WBSProject.from_dict(wbs_project_dict) - wbs_project_json = wbs_project.to_dict() - - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - - # Build the query - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'project_plan.md':\n{project_plan_markdown}\n\n" - f"File 'Work Breakdown Structure.json':\n{format_json_for_use_in_query(wbs_project_json)}\n\n" - f"File 'similar_projects.md':\n{related_resources_markdown}" - ) - - # Execute the pitch creation. - create_pitch = CreatePitch.execute(llm, query) - pitch_dict = create_pitch.raw_response_dict() - - # Write the resulting pitch JSON to the output file. - with self.output().open("w") as f: - json.dump(pitch_dict, f, indent=2) - - logger.info("Pitch created and written to %s", self.output().path) - -class ConvertPitchToMarkdownTask(PlanTask): - """ - Human readable version of the pitch. - - This task depends on: - - CreatePitchTask: Creates the pitch JSON. - """ - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PITCH_CONVERT_TO_MARKDOWN_RAW), - 'markdown': self.local_target(FilenameEnum.PITCH_MARKDOWN) - } - - def requires(self): - return { - 'pitch': self.clone(CreatePitchTask), - } - - def run_with_llm(self, llm: LLM) -> None: - # Read the project plan JSON. - with self.input()['pitch'].open("r") as f: - pitch_json = json.load(f) - - # Build the query - query = format_json_for_use_in_query(pitch_json) - - # Execute the conversion. - converted = ConvertPitchToMarkdown.execute(llm, query) - - # Save the results. - json_path = self.output()['raw'].path - converted.save_raw(json_path) - markdown_path = self.output()['markdown'].path - converted.save_markdown(markdown_path) - - -class IdentifyTaskDependenciesTask(PlanTask): - """ - This task identifies the dependencies between WBS tasks. - """ - def output(self): - return self.local_target(FilenameEnum.TASK_DEPENDENCIES_RAW) - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'wbs_level2': self.clone(CreateWBSLevel2Task), - 'data_collection': self.clone(DataCollectionTask), - } - - def run_with_llm(self, llm: LLM) -> None: - logger.info("Identifying task dependencies...") - - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['wbs_level2']['clean'].open("r") as f: - major_phases_with_subtasks = json.load(f) - - # Build the query - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'project_plan.md':\n{project_plan_markdown}\n\n" - f"File 'Work Breakdown Structure.json':\n{format_json_for_use_in_query(major_phases_with_subtasks)}\n\n" - f"File 'data_collection.md':\n{data_collection_markdown}" - ) - - # Execute the dependency identification. - identify_dependencies = IdentifyWBSTaskDependencies.execute(llm, query) - dependencies_raw_dict = identify_dependencies.raw_response_dict() - - # Write the raw dependencies JSON to the output file. - with self.output().open("w") as f: - json.dump(dependencies_raw_dict, f, indent=2) - - logger.info("Task dependencies identified and written to %s", self.output().path) - -class EstimateTaskDurationsTask(PlanTask): - """ - This task estimates durations for WBS tasks in chunks. - - It depends on: - - ProjectPlanTask: providing the project plan JSON. - - WBSProjectLevel1AndLevel2Task: providing the major phases with subtasks and the task UUIDs. - - For each chunk of 3 task IDs, a raw JSON file (e.g. "011-1-task_durations_raw.json") is written, - and an aggregated JSON file (defined by FilenameEnum.TASK_DURATIONS) is produced. - - IDEA: 1st estimate the Tasks that have zero children. - 2nd estimate tasks that have children where all children have been estimated. - repeat until all tasks have been estimated. - """ - def output(self): - return self.local_target(FilenameEnum.TASK_DURATIONS) - - def requires(self): - return { - 'project_plan': self.clone(ProjectPlanTask), - 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - logger.info("Estimating task durations...") - - # Load the project plan JSON. - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - - with self.input()['wbs_project'].open("r") as f: - wbs_project_dict = json.load(f) - wbs_project = WBSProject.from_dict(wbs_project_dict) - - # json'ish representation of the major phases in the WBS, and their subtasks. - root_task = wbs_project.root_task - major_tasks = [child.to_dict() for child in root_task.task_children] - major_phases_with_subtasks = major_tasks - - # Don't include uuid of the root task. It's the child tasks that are of interest to estimate. - decompose_task_id_list = [] - for task in wbs_project.root_task.task_children: - decompose_task_id_list.extend(task.task_ids()) - - logger.info(f"There are {len(decompose_task_id_list)} tasks to be estimated.") - - # Split the task IDs into chunks of 3. - task_ids_chunks = [decompose_task_id_list[i:i + 3] for i in range(0, len(decompose_task_id_list), 3)] - - # In production mode, all chunks are processed. - # In developer mode, truncate to only 2 chunks for fast turnaround cycle. Otherwise LOTS of tasks are to be estimated. - logger.info(f"EstimateTaskDurationsTask.speedvsdetail: {self.speedvsdetail}") - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") - task_ids_chunks = task_ids_chunks[:2] - else: - logger.info("Processing all chunks.") - - # Process each chunk. - accumulated_task_duration_list = [] - for index, task_ids_chunk in enumerate(task_ids_chunks, start=1): - logger.info("Processing chunk %d of %d", index, len(task_ids_chunks)) - - query = EstimateWBSTaskDurations.format_query( - project_plan_dict, - major_phases_with_subtasks, - task_ids_chunk - ) - - # IDEA: If the chunk file already exist, then there is no need to run the LLM again. - def execute_estimate_task_durations(llm: LLM) -> EstimateWBSTaskDurations: - return EstimateWBSTaskDurations.execute(llm, query) - - try: - estimate_durations = llm_executor.run(execute_estimate_task_durations) - except PipelineStopRequested: - # Re-raise PipelineStopRequested without wrapping it - raise - except Exception as e: - logger.error(f"Task durations chunk {index} LLM interaction failed.", exc_info=True) - raise ValueError(f"Task durations chunk {index} LLM interaction failed.") from e - - durations_raw_dict = estimate_durations.raw_response_dict() - - # Write the raw JSON for this chunk. - filename = FilenameEnum.TASK_DURATIONS_RAW_TEMPLATE.format(index) - raw_chunk_path = self.run_id_dir / filename - with open(raw_chunk_path, "w") as f: - json.dump(durations_raw_dict, f, indent=2) - - accumulated_task_duration_list.extend(durations_raw_dict.get('task_details', [])) - - # Write the aggregated task durations. - aggregated_path = self.file_path(FilenameEnum.TASK_DURATIONS) - with open(aggregated_path, "w") as f: - json.dump(accumulated_task_duration_list, f, indent=2) - - logger.info("Task durations estimated and aggregated results written to %s", aggregated_path) - -class CreateWBSLevel3Task(PlanTask): - """ - This task creates the Work Breakdown Structure (WBS) Level 3, by decomposing tasks from Level 2 into subtasks. - - It depends on: - - ProjectPlanTask: provides the project plan JSON. - - WBSProjectLevel1AndLevel2Task: provides the major phases with subtasks and the task UUIDs. - - EstimateTaskDurationsTask: provides the aggregated task durations (task_duration_list). - - For each task without any subtasks, a query is built and executed using the LLM. - The raw JSON result for each task is written to a file using the template from FilenameEnum. - Finally, all individual results are accumulated and written as an aggregated JSON file. - """ - def output(self): - return self.local_target(FilenameEnum.WBS_LEVEL3) - - def requires(self): - return { - 'project_plan': self.clone(ProjectPlanTask), - 'wbs_project': self.clone(WBSProjectLevel1AndLevel2Task), - 'task_durations': self.clone(EstimateTaskDurationsTask), - 'data_collection': self.clone(DataCollectionTask), - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - logger.info("Creating Work Breakdown Structure (WBS) Level 3...") - - # Read inputs from required tasks. - with self.input()['project_plan']['raw'].open("r") as f: - project_plan_dict = json.load(f) - - with self.input()['wbs_project'].open("r") as f: - wbs_project_dict = json.load(f) - wbs_project = WBSProject.from_dict(wbs_project_dict) - - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - - # Load the estimated task durations. - task_duration_list_path = self.input()['task_durations'].path - WBSPopulate.extend_project_with_durations_json(wbs_project, task_duration_list_path) - # for each task in the wbs_project, find the task that has no children - tasks_with_no_children = [] - def visit_task(task): - if len(task.task_children) == 0: - tasks_with_no_children.append(task) - else: - for child in task.task_children: - visit_task(child) - visit_task(wbs_project.root_task) - - # for each task with no children, extract the task_id - decompose_task_id_list = [] - for task in tasks_with_no_children: - decompose_task_id_list.append(task.id) - - logger.info("There are %d tasks to be decomposed.", len(decompose_task_id_list)) - - # In production mode, all chunks are processed. - # In developer mode, truncate to only 2 chunks for fast turnaround cycle. Otherwise LOTS of tasks are to be decomposed. - logger.info(f"CreateWBSLevel3Task.speedvsdetail: {self.speedvsdetail}") - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 chunks for testing.") - decompose_task_id_list = decompose_task_id_list[:2] - else: - logger.info("Processing all chunks.") - - project_plan_str = format_json_for_use_in_query(project_plan_dict) - wbs_project_str = format_json_for_use_in_query(wbs_project.to_dict()) - - # Loop over each task ID. - wbs_level3_result_accumulated = [] - total_tasks = len(decompose_task_id_list) - for index, task_id in enumerate(decompose_task_id_list, start=1): - logger.info("Decomposing task %d of %d", index, total_tasks) - - query = ( - f"The project plan:\n{project_plan_str}\n\n" - f"Data collection:\n{data_collection_markdown}\n\n" - f"Work breakdown structure:\n{wbs_project_str}\n\n" - f"Only decompose this task:\n\"{task_id}\"" - ) - - # IDEA: If the chunk file already exist, then there is no need to run the LLM again. - def execute_create_wbs_level3(llm: LLM) -> CreateWBSLevel3: - return CreateWBSLevel3.execute(llm, query, task_id) - - try: - create_wbs_level3 = llm_executor.run(execute_create_wbs_level3) - except PipelineStopRequested: - # Re-raise PipelineStopRequested without wrapping it - raise - except Exception as e: - logger.error(f"WBS Level 3 task {index} LLM interaction failed.", exc_info=True) - raise ValueError(f"WBS Level 3 task {index} LLM interaction failed.") from e - - wbs_level3_raw_dict = create_wbs_level3.raw_response_dict() - - # Write the raw JSON for this task using the FilenameEnum template. - raw_filename = FilenameEnum.WBS_LEVEL3_RAW_TEMPLATE.value.format(index) - raw_chunk_path = self.run_id_dir / raw_filename - with open(raw_chunk_path, 'w') as f: - json.dump(wbs_level3_raw_dict, f, indent=2) - - # Accumulate the decomposed tasks. - wbs_level3_result_accumulated.extend(create_wbs_level3.tasks) - - # Write the aggregated WBS Level 3 result. - aggregated_path = self.file_path(FilenameEnum.WBS_LEVEL3) - with open(aggregated_path, 'w') as f: - json.dump(wbs_level3_result_accumulated, f, indent=2) - - logger.info("WBS Level 3 created and aggregated results written to %s", aggregated_path) - -class WBSProjectLevel1AndLevel2AndLevel3Task(PlanTask): - """ - Create a WBS project from the WBS Level 1 and Level 2 and Level 3 JSON files. - - It depends on: - - WBSProjectLevel1AndLevel2Task: providing the major phases with subtasks and the task UUIDs. - - CreateWBSLevel3Task: providing the decomposed tasks. - """ - def output(self): - return { - 'full': self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2_AND_LEVEL3_FULL), - 'csv': self.local_target(FilenameEnum.WBS_PROJECT_LEVEL1_AND_LEVEL2_AND_LEVEL3_CSV) - } - - def requires(self): - return { - 'wbs_project12': self.clone(WBSProjectLevel1AndLevel2Task), - 'wbs_level3': self.clone(CreateWBSLevel3Task), - } - - def run_inner(self): - wbs_project_path = self.input()['wbs_project12'].path - with open(wbs_project_path, "r") as f: - wbs_project_dict = json.load(f) - wbs_project = WBSProject.from_dict(wbs_project_dict) - - wbs_level3_path = self.input()['wbs_level3'].path - WBSPopulate.extend_project_with_decomposed_tasks_json(wbs_project, wbs_level3_path) - - json_representation = json.dumps(wbs_project.to_dict(), indent=2) - with self.output()['full'].open("w") as f: - f.write(json_representation) - - csv_representation = wbs_project.to_csv_string() - with self.output()['csv'].open("w") as f: - f.write(csv_representation) - -class CreateScheduleTask(PlanTask): - def output(self): - return { - 'dhtmlx_html': self.local_target(FilenameEnum.SCHEDULE_GANTT_DHTMLX_HTML), - 'machai_csv': self.local_target(FilenameEnum.SCHEDULE_GANTT_MACHAI_CSV) - } - - def requires(self): - return { - 'start_time': self.clone(StartTimeTask), - 'wbs_level1': self.clone(CreateWBSLevel1Task), - 'dependencies': self.clone(IdentifyTaskDependenciesTask), - 'durations': self.clone(EstimateTaskDurationsTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task) - } - - def run_inner(self): - # For the report title, use the 'project_title' of the WBS Level 1 result. - with self.input()['wbs_level1']['project_title'].open("r") as f: - title = f.read() - with self.input()['dependencies'].open("r") as f: - dependencies_dict = json.load(f) - with self.input()['durations'].open("r") as f: - duration_list: list[dict[str, Any]] = json.load(f) - wbs_project_path = self.input()['wbs_project123']['full'].path - with open(wbs_project_path, "r") as f: - wbs_project_dict = json.load(f) - wbs_project = WBSProject.from_dict(wbs_project_dict) - - # Read the start time from the StartTimeTask to get the actual pipeline start date - with self.input()['start_time'].open("r") as f: - start_time_dict = json.load(f) - - # The start_time.server_iso_utc is in format "YYYY-MM-DDTHH:MM:SSZ" - utc_timestamp = start_time_dict.get('server_iso_utc') or start_time_dict.get('utc_timestamp', '') - # The 'Z' suffix for UTC is not supported by fromisoformat() in Python < 3.11. Replace, ensures compatibility. - project_start_dt: datetime = datetime.fromisoformat(utc_timestamp.replace('Z', '+00:00')) - project_start: date = project_start_dt.date() - - # logger.debug(f"dependencies_dict {dependencies_dict}") - # logger.debug(f"duration_list {duration_list}") - # logger.debug(f"wbs_project {wbs_project.to_dict()}") - - # Tooltips with a detailed description of each task. - task_id_to_html_tooltip_dict: dict[str, str] = WBSTaskTooltip.html_tooltips(wbs_project) - task_id_to_text_tooltip_dict: dict[str, str] = WBSTaskTooltip.text_tooltips(wbs_project) + def local_target(self, filename: FilenameEnum) -> luigi.LocalTarget: + return luigi.LocalTarget(self.file_path(filename)) - project_schedule: ProjectSchedule = ProjectSchedulePopulator.populate( - wbs_project=wbs_project, - duration_list=duration_list - ) + def create_llm_executor(self) -> LLMExecutor: + """ + Create an LLMExecutor instance. + - Responsible for stopping the pipeline when the user presses Ctrl-C or closes the browser tab. + - Fallback mechanism to try the next LLM if the current one fails. + """ + # Redirect the callback to the pipeline_executor_callback. + def should_stop_callback(parameters: ShouldStopCallbackParameters) -> None: + if self._pipeline_executor_callback is None: + return + # The pipeline_executor_callback expects (task, duration) but we have ShouldStopCallbackParameters + total_duration = parameters.total_duration + self._pipeline_executor_callback(self, total_duration) - # Export the Gantt chart to CSV. - # Always run the CSV export so that the code gets exercised, otherwise the code will rot. - csv_data: str = ExportGanttCSV.to_gantt_csv( - project_schedule=project_schedule, - project_start=project_start, - task_id_to_tooltip_dict=task_id_to_text_tooltip_dict - ) - if PIPELINE_CONFIG.enable_csv_export == False: - # When disabled, then hide the "Export to CSV" button and don't embed the CSV data in the html report. - csv_data = None + llm_model_instances = LLMModelFromName.from_names(self.llm_models) - ExportGanttCSV.save( - project_schedule=project_schedule, - path=self.output()['machai_csv'].path, - project_start=project_start, - task_id_to_tooltip_dict=task_id_to_text_tooltip_dict + return LLMExecutor( + llm_models=llm_model_instances, + should_stop_callback=should_stop_callback, + retry_config=RetryConfig(), ) - # Identify the tasks that should be treated as project activities. - task_ids_to_treat_as_project_activities = wbs_project.task_ids_with_one_or_more_children() - - # Export the Gantt chart to Frappe. - # I'm disappointed by Frappe, it lacks a lot of features that are present in DHTMLX. - # ExportGanttFrappe.save( - # project_schedule=project_schedule, - # path=self.output()['frappe_html'].path, - # project_start=project_start, - # task_ids_to_treat_as_project_activities=task_ids_to_treat_as_project_activities - # ) - - # Export the Gantt chart to DHTMLX. - ExportGanttDHTMLX.save( - project_schedule=project_schedule, - path=self.output()['dhtmlx_html'].path, - project_start=project_start, - task_ids_to_treat_as_project_activities=task_ids_to_treat_as_project_activities, - task_id_to_tooltip_dict=task_id_to_html_tooltip_dict, - title=title, - csv_data=csv_data - ) + def run(self): + """ + Don't override this method. Instead either override the run_inner() method, or override the run_with_llm() method. + """ + try: + self.run_inner() + except PipelineStopRequested as e: + logger.debug(f"{self.__class__.__name__} -> PipelineStopRequested raised: {e}") + # This exception is raised by the should_stop_callback + # If we get here, it means that the pipeline was aborted by the callback, such as by the user pressing Ctrl-C or closing the browser tab. + # Create a flag file to signal that the stop was intentional. + flag_path = self.run_id_dir / ExtraFilenameEnum.PIPELINE_STOP_REQUESTED_FLAG.value + logger.info(f"Creating stop flag file: {flag_path!r}") + flag_path.touch() + raise + except Exception as e: + # Re-raise the exception with a more descriptive message + raise Exception(f"Failed to run {self.__class__.__name__} with any of the LLMs in the list: {self.llm_models!r} for run_id_dir: {self.run_id_dir!r}") from e -class ReviewPlanTask(PlanTask): - """ - Ask questions about the almost finished plan. - """ - def output(self): - return { - 'raw': self.local_target(FilenameEnum.REVIEW_PLAN_RAW), - 'markdown': self.local_target(FilenameEnum.REVIEW_PLAN_MARKDOWN) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'data_collection': self.clone(DataCollectionTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'expert_review': self.clone(ExpertReviewTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task) - } - def run_inner(self): + """ + Override this method or the run_with_llm() method. + """ llm_executor: LLMExecutor = self.create_llm_executor() - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['pitch_markdown']['markdown'].open("r") as f: - pitch_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - with self.input()['wbs_project123']['csv'].open("r") as f: - wbs_project_csv = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'data-collection.md':\n{data_collection_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'pitch.md':\n{pitch_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}\n\n" - f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}" - ) - - # Perform the review. - review_plan = ReviewPlan.execute(llm_executor=llm_executor, document=query, speed_vs_detail=self.speedvsdetail) - - # Save the results. - json_path = self.output()['raw'].path - review_plan.save_raw(json_path) - markdown_path = self.output()['markdown'].path - review_plan.save_markdown(markdown_path) - - -class ExecutiveSummaryTask(PlanTask): - """ - Create an executive summary of the plan. - """ - def output(self): - return { - 'raw': self.local_target(FilenameEnum.EXECUTIVE_SUMMARY_RAW), - 'markdown': self.local_target(FilenameEnum.EXECUTIVE_SUMMARY_MARKDOWN) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'project_plan': self.clone(ProjectPlanTask), - 'data_collection': self.clone(DataCollectionTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'expert_review': self.clone(ExpertReviewTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'review_plan': self.clone(ReviewPlanTask) - } - - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['pitch_markdown']['markdown'].open("r") as f: - pitch_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - with self.input()['wbs_project123']['csv'].open("r") as f: - wbs_project_csv = f.read() - with self.input()['review_plan']['markdown'].open("r") as f: - review_plan_markdown = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'data-collection.md':\n{data_collection_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'pitch.md':\n{pitch_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}\n\n" - f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" - f"File 'review-plan.md':\n{review_plan_markdown}" - ) - - # Create the executive summary. - executive_summary = ExecutiveSummary.execute(llm, query) - - # Save the results. - json_path = self.output()['raw'].path - executive_summary.save_raw(json_path) - markdown_path = self.output()['markdown'].path - executive_summary.save_markdown(markdown_path) + # Attempt executing this code with the first LLM, if that fails, try the next one, and so on. + def execute_function(llm: LLM) -> None: + self.run_with_llm(llm) + # Make multiple attempts at running the run_with_llm() function. + # No try/except needed here. Let PlanTask.run() handle it. + llm_executor.run(execute_function) -class QuestionsAndAnswersTask(PlanTask): - def output(self): - return { - 'raw': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_RAW), - 'markdown': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_MARKDOWN), - 'html': self.local_target(FilenameEnum.QUESTIONS_AND_ANSWERS_HTML) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'consolidate_governance': self.clone(ConsolidateGovernanceTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'data_collection': self.clone(DataCollectionTask), - 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'expert_review': self.clone(ExpertReviewTask), - 'project_plan': self.clone(ProjectPlanTask), - 'review_plan': self.clone(ReviewPlanTask), - } - def run_with_llm(self, llm: LLM) -> None: - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['pitch_markdown']['markdown'].open("r") as f: - pitch_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - with self.input()['wbs_project123']['csv'].open("r") as f: - wbs_project_csv = f.read() - with self.input()['review_plan']['markdown'].open("r") as f: - review_plan_markdown = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'data-collection.md':\n{data_collection_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'pitch.md':\n{pitch_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}\n\n" - f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" - f"File 'review-plan.md':\n{review_plan_markdown}" - ) - - # Invoke the LLM - question_answers = QuestionsAnswers.execute(llm, query) - - # Save the results. - json_path = self.output()['raw'].path - question_answers.save_raw(json_path) - markdown_path = self.output()['markdown'].path - question_answers.save_markdown(markdown_path) - html_path = self.output()['html'].path - question_answers.save_html(html_path) - -class PremortemTask(PlanTask): - def output(self): - return { - 'raw': self.local_target(FilenameEnum.PREMORTEM_RAW), - 'markdown': self.local_target(FilenameEnum.PREMORTEM_MARKDOWN) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'consolidate_governance': self.clone(ConsolidateGovernanceTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'data_collection': self.clone(DataCollectionTask), - 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'expert_review': self.clone(ExpertReviewTask), - 'project_plan': self.clone(ProjectPlanTask), - 'review_plan': self.clone(ReviewPlanTask), - 'questions_and_answers': self.clone(QuestionsAndAnswersTask) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['pitch_markdown']['markdown'].open("r") as f: - pitch_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - with self.input()['wbs_project123']['csv'].open("r") as f: - wbs_project_csv = f.read() - with self.input()['review_plan']['markdown'].open("r") as f: - review_plan_markdown = f.read() - with self.input()['questions_and_answers']['markdown'].open("r") as f: - questions_and_answers_markdown = f.read() - - # Build the query. - query = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'data-collection.md':\n{data_collection_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'pitch.md':\n{pitch_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}\n\n" - f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" - f"File 'review-plan.md':\n{review_plan_markdown}\n\n" - f"File 'questions-and-answers.md':\n{questions_and_answers_markdown}" - ) - - # Invoke the LLM - premortem = Premortem.execute(llm_executor=llm_executor, speed_vs_detail=self.speedvsdetail, user_prompt=query) - - # Save the results. - json_path = self.output()['raw'].path - premortem.save_raw(json_path) - markdown_path = self.output()['markdown'].path - premortem.save_markdown(markdown_path) - - -class SelfAuditTask(PlanTask): - def output(self): - return { - 'raw': self.local_target(FilenameEnum.SELF_AUDIT_RAW), - 'markdown': self.local_target(FilenameEnum.SELF_AUDIT_MARKDOWN) - } - - def requires(self): - return { - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'consolidate_governance': self.clone(ConsolidateGovernanceTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'data_collection': self.clone(DataCollectionTask), - 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'expert_review': self.clone(ExpertReviewTask), - 'project_plan': self.clone(ProjectPlanTask), - 'review_plan': self.clone(ReviewPlanTask), - 'questions_and_answers': self.clone(QuestionsAndAnswersTask), - 'premortem': self.clone(PremortemTask) - } - - def run_inner(self): - llm_executor: LLMExecutor = self.create_llm_executor() - - # Read inputs from required tasks. - with self.input()['strategic_decisions_markdown']['markdown'].open("r") as f: - strategic_decisions_markdown = f.read() - with self.input()['scenarios_markdown']['markdown'].open("r") as f: - scenarios_markdown = f.read() - with self.input()['consolidate_assumptions_markdown']['short'].open("r") as f: - assumptions_markdown = f.read() - with self.input()['project_plan']['markdown'].open("r") as f: - project_plan_markdown = f.read() - with self.input()['data_collection']['markdown'].open("r") as f: - data_collection_markdown = f.read() - with self.input()['related_resources']['markdown'].open("r") as f: - related_resources_markdown = f.read() - with self.input()['swot_analysis']['markdown'].open("r") as f: - swot_analysis_markdown = f.read() - with self.input()['team_markdown'].open("r") as f: - team_markdown = f.read() - with self.input()['pitch_markdown']['markdown'].open("r") as f: - pitch_markdown = f.read() - with self.input()['expert_review'].open("r") as f: - expert_review = f.read() - with self.input()['wbs_project123']['csv'].open("r") as f: - wbs_project_csv = f.read() - with self.input()['review_plan']['markdown'].open("r") as f: - review_plan_markdown = f.read() - with self.input()['questions_and_answers']['markdown'].open("r") as f: - questions_and_answers_markdown = f.read() - with self.input()['premortem']['markdown'].open("r") as f: - premortem_markdown = f.read() - - # Build the query. - user_prompt = ( - f"File 'strategic_decisions.md':\n{strategic_decisions_markdown}\n\n" - f"File 'scenarios.md':\n{scenarios_markdown}\n\n" - f"File 'assumptions.md':\n{assumptions_markdown}\n\n" - f"File 'project-plan.md':\n{project_plan_markdown}\n\n" - f"File 'data-collection.md':\n{data_collection_markdown}\n\n" - f"File 'related-resources.md':\n{related_resources_markdown}\n\n" - f"File 'swot-analysis.md':\n{swot_analysis_markdown}\n\n" - f"File 'team.md':\n{team_markdown}\n\n" - f"File 'pitch.md':\n{pitch_markdown}\n\n" - f"File 'expert-review.md':\n{expert_review}\n\n" - f"File 'work-breakdown-structure.csv':\n{wbs_project_csv}\n\n" - f"File 'review-plan.md':\n{review_plan_markdown}\n\n" - f"File 'questions-and-answers.md':\n{questions_and_answers_markdown}\n\n" - f"File 'premortem.md':\n{premortem_markdown}" - ) - - logger.info(f"SelfAuditTask.speedvsdetail: {self.speedvsdetail}") - max_number_of_items: Optional[int] = None - if self.speedvsdetail == SpeedVsDetailEnum.FAST_BUT_SKIP_DETAILS: - logger.info("FAST_BUT_SKIP_DETAILS mode, truncating to 2 items for testing a subset of the SelfAudit items.") - max_number_of_items = 2 - else: - logger.info("Processing all SelfAudit items.") - - # Invoke the LLM - self_audit = SelfAudit.execute( - llm_executor=llm_executor, - user_prompt=user_prompt, - max_number_of_items=max_number_of_items, - ) - - # Save the results. - json_path = self.output()['raw'].path - self_audit.save_raw(json_path) - markdown_path = self.output()['markdown'].path - self_audit.save_markdown(markdown_path) - -class ReportTask(PlanTask): - """ - Generate a report html document. - """ - def output(self): - return self.local_target(FilenameEnum.REPORT) - - def requires(self): - return { - 'setup': self.clone(SetupTask), - 'redline_gate': self.clone(RedlineGateTask), - 'premise_attack': self.clone(PremiseAttackTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'consolidate_governance': self.clone(ConsolidateGovernanceTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'data_collection': self.clone(DataCollectionTask), - 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), - 'wbs_level1': self.clone(CreateWBSLevel1Task), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'expert_review': self.clone(ExpertReviewTask), - 'project_plan': self.clone(ProjectPlanTask), - 'review_plan': self.clone(ReviewPlanTask), - 'executive_summary': self.clone(ExecutiveSummaryTask), - 'create_schedule': self.clone(CreateScheduleTask), - 'questions_and_answers': self.clone(QuestionsAndAnswersTask), - 'premortem': self.clone(PremortemTask), - 'self_audit': self.clone(SelfAuditTask) - } - - def run_inner(self): - # For the report title, use the 'project_title' of the WBS Level 1 result. - with self.input()['wbs_level1']['project_title'].open("r") as f: - title = f.read() - - rg = ReportGenerator() - rg.append_markdown('Executive Summary', self.input()['executive_summary']['markdown'].path) - rg.append_html('Gantt Interactive', self.input()['create_schedule']['dhtmlx_html'].path) - rg.append_markdown('Pitch', self.input()['pitch_markdown']['markdown'].path) - rg.append_markdown('Project Plan', self.input()['project_plan']['markdown'].path) - rg.append_markdown('Strategic Decisions', self.input()['strategic_decisions_markdown']['markdown'].path) - rg.append_markdown('Scenarios', self.input()['scenarios_markdown']['markdown'].path) - rg.append_markdown('Assumptions', self.input()['consolidate_assumptions_markdown']['full'].path) - rg.append_markdown('Governance', self.input()['consolidate_governance'].path) - rg.append_markdown('Related Resources', self.input()['related_resources']['markdown'].path) - rg.append_markdown('Data Collection', self.input()['data_collection']['markdown'].path) - rg.append_markdown('Documents to Create and Find', self.input()['documents_to_create_and_find'].path) - rg.append_markdown('SWOT Analysis', self.input()['swot_analysis']['markdown'].path) - rg.append_markdown('Team', self.input()['team_markdown'].path) - rg.append_markdown('Expert Criticism', self.input()['expert_review'].path) - rg.append_csv('Work Breakdown Structure', self.input()['wbs_project123']['csv'].path) - rg.append_markdown('Review Plan', self.input()['review_plan']['markdown'].path) - rg.append_html('Questions & Answers', self.input()['questions_and_answers']['html'].path) - rg.append_markdown_with_tables('Premortem', self.input()['premortem']['markdown'].path) - rg.append_markdown_with_tables('Self Audit', self.input()['self_audit']['markdown'].path) - rg.append_initial_prompt_vetted( - document_title='Initial Prompt Vetted', - initial_prompt_file_path=self.input()['setup'].path, - redline_gate_markdown_file_path=self.input()['redline_gate']['markdown'].path, - premise_attack_markdown_file_path=self.input()['premise_attack']['markdown'].path - ) - rg.save_report(self.output().path, title=title, execute_plan_section_hidden=REPORT_EXECUTE_PLAN_SECTION_HIDDEN) - -class FullPlanPipeline(PlanTask): - def requires(self): - return { - 'start_time': self.clone(StartTimeTask), - 'setup': self.clone(SetupTask), - 'redline_gate': self.clone(RedlineGateTask), - 'premise_attack': self.clone(PremiseAttackTask), - 'identify_purpose': self.clone(IdentifyPurposeTask), - 'plan_type': self.clone(PlanTypeTask), - 'potential_levers': self.clone(PotentialLeversTask), - 'deduplicate_levers': self.clone(DeduplicateLeversTask), - 'enriched_levers': self.clone(EnrichLeversTask), - 'focus_on_vital_few_levers': self.clone(FocusOnVitalFewLeversTask), - 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), - 'candidate_scenarios': self.clone(CandidateScenariosTask), - 'select_scenario': self.clone(SelectScenarioTask), - 'scenarios_markdown': self.clone(ScenariosMarkdownTask), - 'physical_locations': self.clone(PhysicalLocationsTask), - 'currency_strategy': self.clone(CurrencyStrategyTask), - 'identify_risks': self.clone(IdentifyRisksTask), - 'make_assumptions': self.clone(MakeAssumptionsTask), - 'assumptions': self.clone(DistillAssumptionsTask), - 'review_assumptions': self.clone(ReviewAssumptionsTask), - 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), - 'pre_project_assessment': self.clone(PreProjectAssessmentTask), - 'project_plan': self.clone(ProjectPlanTask), - 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), - 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), - 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), - 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), - 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask), - 'governance_phase6_extra': self.clone(GovernancePhase6ExtraTask), - 'consolidate_governance': self.clone(ConsolidateGovernanceTask), - 'related_resources': self.clone(RelatedResourcesTask), - 'find_team_members': self.clone(FindTeamMembersTask), - 'enrich_team_members_with_contract_type': self.clone(EnrichTeamMembersWithContractTypeTask), - 'enrich_team_members_with_background_story': self.clone(EnrichTeamMembersWithBackgroundStoryTask), - 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), - 'review_team': self.clone(ReviewTeamTask), - 'team_markdown': self.clone(TeamMarkdownTask), - 'swot_analysis': self.clone(SWOTAnalysisTask), - 'expert_review': self.clone(ExpertReviewTask), - 'data_collection': self.clone(DataCollectionTask), - 'identified_documents': self.clone(IdentifyDocumentsTask), - 'filter_documents_to_find': self.clone(FilterDocumentsToFindTask), - 'filter_documents_to_create': self.clone(FilterDocumentsToCreateTask), - 'draft_documents_to_find': self.clone(DraftDocumentsToFindTask), - 'draft_documents_to_create': self.clone(DraftDocumentsToCreateTask), - 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), - 'wbs_level1': self.clone(CreateWBSLevel1Task), - 'wbs_level2': self.clone(CreateWBSLevel2Task), - 'wbs_project12': self.clone(WBSProjectLevel1AndLevel2Task), - 'pitch_raw': self.clone(CreatePitchTask), - 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), - 'dependencies': self.clone(IdentifyTaskDependenciesTask), - 'durations': self.clone(EstimateTaskDurationsTask), - 'wbs_level3': self.clone(CreateWBSLevel3Task), - 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), - 'plan_evaluator': self.clone(ReviewPlanTask), - 'executive_summary': self.clone(ExecutiveSummaryTask), - 'create_schedule': self.clone(CreateScheduleTask), - 'questions_and_answers': self.clone(QuestionsAndAnswersTask), - 'premortem': self.clone(PremortemTask), - 'self_audit': self.clone(SelfAuditTask), - 'report': self.clone(ReportTask), - } + """ + Override this method or the run_inner() method. + """ + raise NotImplementedError("Subclasses must implement this method.") - def output(self): - return self.local_target(FilenameEnum.PIPELINE_COMPLETE) - def run_inner(self): - with self.output().open("w") as f: - f.write("Full pipeline executed successfully.\n") +# --------------------------------------------------------------------------- +# Task class definitions have been extracted to individual files under +# worker_plan_internal/plan/stages/. The pipeline orchestrator that wires +# them together lives in stages/full_plan_pipeline.py. +# --------------------------------------------------------------------------- def _task_class_to_step_label(class_name: str) -> str: @@ -3884,7 +189,7 @@ class ExecutePipeline: speedvsdetail: SpeedVsDetailEnum llm_models: list[str] model_profile: ModelProfileEnum = ModelProfileEnum.BASELINE - full_plan_pipeline_task: Optional[FullPlanPipeline] = field(default=None) + full_plan_pipeline_task: Optional[Any] = field(default=None) all_expected_filenames: list[str] = field(default_factory=list) luigi_build_return_value: Optional[bool] = field(default=None, init=False) @@ -3899,10 +204,11 @@ def setup(self) -> None: if not (self.run_id_dir / FilenameEnum.INITIAL_PLAN.value).exists(): raise FileNotFoundError(f"The '{FilenameEnum.INITIAL_PLAN.value}' file does not exist in the run_id_dir: {self.run_id_dir!r}") + from worker_plan_internal.plan.stages.full_plan_pipeline import FullPlanPipeline full_plan_pipeline_task = FullPlanPipeline( - run_id_dir=self.run_id_dir, - speedvsdetail=self.speedvsdetail, - llm_models=self.llm_models, + run_id_dir=self.run_id_dir, + speedvsdetail=self.speedvsdetail, + llm_models=self.llm_models, _pipeline_executor_callback=self.callback_run_task ) self.full_plan_pipeline_task = full_plan_pipeline_task @@ -3942,7 +248,7 @@ def resolve_luigi_workers(self) -> int: if not workers_candidates: return default_workers return min(workers_candidates) - + @classmethod def resolve_llm_models( cls, @@ -3973,7 +279,7 @@ def resolve_llm_models( for index, llm_name in enumerate(llm_models): logger.info(f"{index}. {llm_name!r}") return llm_models - + def get_progress_percentage(self) -> PipelineProgress: files = [] try: @@ -4162,7 +468,7 @@ def configure_logging(run_id_dir: Path) -> int: logger.error(msg) print(f"Exiting... {msg}") sys.exit(1) - + # Initialize token tracking with the task identifier. task_id = os.environ.get("PLANEXE_TASK_ID") set_current_task_id(task_id) @@ -4241,15 +547,15 @@ def configure_logging(run_id_dir: Path) -> int: llm_models=llm_models, model_profile=model_profile, ) - + try: execute_pipeline.setup() except Exception as e: logger.error(f"Failed to setup pipeline: {e}") sys.exit(1) - + logger.info(f"execute_pipeline: {execute_pipeline!r}") - + try: execute_pipeline.run() except Exception as e: diff --git a/worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py b/worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py new file mode 100644 index 000000000..6caf9e6d1 --- /dev/null +++ b/worker_plan/worker_plan_internal/plan/stages/full_plan_pipeline.py @@ -0,0 +1,156 @@ +"""Pipeline orchestrator: declares all stages as Luigi dependencies.""" +from worker_plan_internal.plan.run_plan_pipeline import PlanTask +from worker_plan_api.filenames import FilenameEnum + +# Phase 1-2 +from worker_plan_internal.plan.stages.start_time import StartTimeTask +from worker_plan_internal.plan.stages.setup import SetupTask +from worker_plan_internal.plan.stages.redline_gate import RedlineGateTask +from worker_plan_internal.plan.stages.premise_attack import PremiseAttackTask +from worker_plan_internal.plan.stages.identify_purpose import IdentifyPurposeTask +from worker_plan_internal.plan.stages.plan_type import PlanTypeTask + +# Phase 3 +from worker_plan_internal.plan.stages.potential_levers import PotentialLeversTask +from worker_plan_internal.plan.stages.deduplicate_levers import DeduplicateLeversTask +from worker_plan_internal.plan.stages.enrich_levers import EnrichLeversTask +from worker_plan_internal.plan.stages.focus_on_vital_few_levers import FocusOnVitalFewLeversTask +from worker_plan_internal.plan.stages.strategic_decisions_markdown import StrategicDecisionsMarkdownTask +from worker_plan_internal.plan.stages.candidate_scenarios import CandidateScenariosTask +from worker_plan_internal.plan.stages.select_scenario import SelectScenarioTask +from worker_plan_internal.plan.stages.scenarios_markdown import ScenariosMarkdownTask + +# Phase 4-5 +from worker_plan_internal.plan.stages.physical_locations import PhysicalLocationsTask +from worker_plan_internal.plan.stages.currency_strategy import CurrencyStrategyTask +from worker_plan_internal.plan.stages.identify_risks import IdentifyRisksTask +from worker_plan_internal.plan.stages.make_assumptions import MakeAssumptionsTask +from worker_plan_internal.plan.stages.distill_assumptions import DistillAssumptionsTask +from worker_plan_internal.plan.stages.review_assumptions import ReviewAssumptionsTask +from worker_plan_internal.plan.stages.consolidate_assumptions_markdown import ConsolidateAssumptionsMarkdownTask + +# Phase 6-7 +from worker_plan_internal.plan.stages.pre_project_assessment import PreProjectAssessmentTask +from worker_plan_internal.plan.stages.project_plan import ProjectPlanTask +from worker_plan_internal.plan.stages.governance_phase1_audit import GovernancePhase1AuditTask +from worker_plan_internal.plan.stages.governance_phase2_bodies import GovernancePhase2BodiesTask +from worker_plan_internal.plan.stages.governance_phase3_impl_plan import GovernancePhase3ImplPlanTask +from worker_plan_internal.plan.stages.governance_phase4_decision_escalation_matrix import GovernancePhase4DecisionEscalationMatrixTask +from worker_plan_internal.plan.stages.governance_phase5_monitoring_progress import GovernancePhase5MonitoringProgressTask +from worker_plan_internal.plan.stages.governance_phase6_extra import GovernancePhase6ExtraTask +from worker_plan_internal.plan.stages.consolidate_governance import ConsolidateGovernanceTask + +# Phase 8 +from worker_plan_internal.plan.stages.related_resources import RelatedResourcesTask +from worker_plan_internal.plan.stages.find_team_members import FindTeamMembersTask +from worker_plan_internal.plan.stages.enrich_team_contract_type import EnrichTeamMembersWithContractTypeTask +from worker_plan_internal.plan.stages.enrich_team_background_story import EnrichTeamMembersWithBackgroundStoryTask +from worker_plan_internal.plan.stages.enrich_team_environment_info import EnrichTeamMembersWithEnvironmentInfoTask +from worker_plan_internal.plan.stages.review_team import ReviewTeamTask +from worker_plan_internal.plan.stages.team_markdown import TeamMarkdownTask + +# Phase 9-10 +from worker_plan_internal.plan.stages.swot_analysis import SWOTAnalysisTask +from worker_plan_internal.plan.stages.expert_review import ExpertReviewTask +from worker_plan_internal.plan.stages.data_collection import DataCollectionTask +from worker_plan_internal.plan.stages.identify_documents import IdentifyDocumentsTask +from worker_plan_internal.plan.stages.filter_documents_to_find import FilterDocumentsToFindTask +from worker_plan_internal.plan.stages.filter_documents_to_create import FilterDocumentsToCreateTask +from worker_plan_internal.plan.stages.draft_documents_to_find import DraftDocumentsToFindTask +from worker_plan_internal.plan.stages.draft_documents_to_create import DraftDocumentsToCreateTask +from worker_plan_internal.plan.stages.markdown_documents import MarkdownWithDocumentsToCreateAndFindTask + +# Phase 11 +from worker_plan_internal.plan.stages.create_wbs_level1 import CreateWBSLevel1Task +from worker_plan_internal.plan.stages.create_wbs_level2 import CreateWBSLevel2Task +from worker_plan_internal.plan.stages.wbs_project_level1_and_level2 import WBSProjectLevel1AndLevel2Task +from worker_plan_internal.plan.stages.create_pitch import CreatePitchTask +from worker_plan_internal.plan.stages.convert_pitch_to_markdown import ConvertPitchToMarkdownTask +from worker_plan_internal.plan.stages.identify_task_dependencies import IdentifyTaskDependenciesTask +from worker_plan_internal.plan.stages.estimate_task_durations import EstimateTaskDurationsTask +from worker_plan_internal.plan.stages.create_wbs_level3 import CreateWBSLevel3Task +from worker_plan_internal.plan.stages.wbs_project_level1_level2_level3 import WBSProjectLevel1AndLevel2AndLevel3Task + +# Phase 12-13 +from worker_plan_internal.plan.stages.create_schedule import CreateScheduleTask +from worker_plan_internal.plan.stages.review_plan import ReviewPlanTask +from worker_plan_internal.plan.stages.executive_summary import ExecutiveSummaryTask +from worker_plan_internal.plan.stages.questions_and_answers import QuestionsAndAnswersTask +from worker_plan_internal.plan.stages.premortem import PremortemTask +from worker_plan_internal.plan.stages.self_audit import SelfAuditTask +from worker_plan_internal.plan.stages.report import ReportTask + + +class FullPlanPipeline(PlanTask): + def requires(self): + return { + 'start_time': self.clone(StartTimeTask), + 'setup': self.clone(SetupTask), + 'redline_gate': self.clone(RedlineGateTask), + 'premise_attack': self.clone(PremiseAttackTask), + 'identify_purpose': self.clone(IdentifyPurposeTask), + 'plan_type': self.clone(PlanTypeTask), + 'potential_levers': self.clone(PotentialLeversTask), + 'deduplicate_levers': self.clone(DeduplicateLeversTask), + 'enriched_levers': self.clone(EnrichLeversTask), + 'focus_on_vital_few_levers': self.clone(FocusOnVitalFewLeversTask), + 'strategic_decisions_markdown': self.clone(StrategicDecisionsMarkdownTask), + 'candidate_scenarios': self.clone(CandidateScenariosTask), + 'select_scenario': self.clone(SelectScenarioTask), + 'scenarios_markdown': self.clone(ScenariosMarkdownTask), + 'physical_locations': self.clone(PhysicalLocationsTask), + 'currency_strategy': self.clone(CurrencyStrategyTask), + 'identify_risks': self.clone(IdentifyRisksTask), + 'make_assumptions': self.clone(MakeAssumptionsTask), + 'assumptions': self.clone(DistillAssumptionsTask), + 'review_assumptions': self.clone(ReviewAssumptionsTask), + 'consolidate_assumptions_markdown': self.clone(ConsolidateAssumptionsMarkdownTask), + 'pre_project_assessment': self.clone(PreProjectAssessmentTask), + 'project_plan': self.clone(ProjectPlanTask), + 'governance_phase1_audit': self.clone(GovernancePhase1AuditTask), + 'governance_phase2_bodies': self.clone(GovernancePhase2BodiesTask), + 'governance_phase3_impl_plan': self.clone(GovernancePhase3ImplPlanTask), + 'governance_phase4_decision_escalation_matrix': self.clone(GovernancePhase4DecisionEscalationMatrixTask), + 'governance_phase5_monitoring_progress': self.clone(GovernancePhase5MonitoringProgressTask), + 'governance_phase6_extra': self.clone(GovernancePhase6ExtraTask), + 'consolidate_governance': self.clone(ConsolidateGovernanceTask), + 'related_resources': self.clone(RelatedResourcesTask), + 'find_team_members': self.clone(FindTeamMembersTask), + 'enrich_team_members_with_contract_type': self.clone(EnrichTeamMembersWithContractTypeTask), + 'enrich_team_members_with_background_story': self.clone(EnrichTeamMembersWithBackgroundStoryTask), + 'enrich_team_members_with_environment_info': self.clone(EnrichTeamMembersWithEnvironmentInfoTask), + 'review_team': self.clone(ReviewTeamTask), + 'team_markdown': self.clone(TeamMarkdownTask), + 'swot_analysis': self.clone(SWOTAnalysisTask), + 'expert_review': self.clone(ExpertReviewTask), + 'data_collection': self.clone(DataCollectionTask), + 'identified_documents': self.clone(IdentifyDocumentsTask), + 'filter_documents_to_find': self.clone(FilterDocumentsToFindTask), + 'filter_documents_to_create': self.clone(FilterDocumentsToCreateTask), + 'draft_documents_to_find': self.clone(DraftDocumentsToFindTask), + 'draft_documents_to_create': self.clone(DraftDocumentsToCreateTask), + 'documents_to_create_and_find': self.clone(MarkdownWithDocumentsToCreateAndFindTask), + 'wbs_level1': self.clone(CreateWBSLevel1Task), + 'wbs_level2': self.clone(CreateWBSLevel2Task), + 'wbs_project12': self.clone(WBSProjectLevel1AndLevel2Task), + 'pitch_raw': self.clone(CreatePitchTask), + 'pitch_markdown': self.clone(ConvertPitchToMarkdownTask), + 'dependencies': self.clone(IdentifyTaskDependenciesTask), + 'durations': self.clone(EstimateTaskDurationsTask), + 'wbs_level3': self.clone(CreateWBSLevel3Task), + 'wbs_project123': self.clone(WBSProjectLevel1AndLevel2AndLevel3Task), + 'plan_evaluator': self.clone(ReviewPlanTask), + 'executive_summary': self.clone(ExecutiveSummaryTask), + 'create_schedule': self.clone(CreateScheduleTask), + 'questions_and_answers': self.clone(QuestionsAndAnswersTask), + 'premortem': self.clone(PremortemTask), + 'self_audit': self.clone(SelfAuditTask), + 'report': self.clone(ReportTask), + } + + def output(self): + return self.local_target(FilenameEnum.PIPELINE_COMPLETE) + + def run_inner(self): + with self.output().open("w") as f: + f.write("Full pipeline executed successfully.\n") From 19cd352031fbe298ae70e100faf4ce958566e5d7 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 19:17:51 +0200 Subject: [PATCH 11/12] docs: update AGENTS.md and remediation roadmap for pipeline split Co-Authored-By: Claude Opus 4.6 (1M context) --- ...odebase-cleanliness-remediation-roadmap.md | 2 +- worker_plan/AGENTS.md | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/proposals/131-codebase-cleanliness-remediation-roadmap.md b/docs/proposals/131-codebase-cleanliness-remediation-roadmap.md index 6ee3e47b4..418c04a98 100644 --- a/docs/proposals/131-codebase-cleanliness-remediation-roadmap.md +++ b/docs/proposals/131-codebase-cleanliness-remediation-roadmap.md @@ -70,7 +70,7 @@ Large modules make the code harder to reason about, harder to test in isolation, 1. ~~Split `frontend_multi_user/src/app.py` by concern into `auth`, `billing`, `admin`, `downloads`, `account`, and `plan_routes`.~~ **Done** (PR #476): Split 3,857-line monolith into 6 Flask Blueprint modules + utils (app.py reduced to 1,441 lines). Follow-up fix: updated all `url_for()` calls in templates to use blueprint-prefixed endpoint names (`plan_routes.*`, `auth.*`, `downloads.*`). 2. ~~Split `mcp_cloud/http_server.py` into `middleware`, `route_registration`, `tool_http_bridge`, and `server_boot`.~~ **Done**: Split 1,439-line monolith into 4 focused modules + re-export shim. -3. Convert `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` from a giant task registry file into a thin pipeline assembly module plus task-specific modules grouped by stage. +3. ~~Convert `worker_plan/worker_plan_internal/plan/run_plan_pipeline.py` from a giant task registry file into a thin pipeline assembly module plus task-specific modules grouped by stage.~~ **Done**: Split 4,257-line monolith into ~66 individual stage files under `stages/` + framework-only core module (563 lines). 4. Extract reusable orchestration helpers from `worker_plan_database/app.py` into focused worker, billing, and queue modules. 5. Set an internal size target for service modules. As a starting rule, new files should stay below roughly 500 lines unless there is a strong reason not to. diff --git a/worker_plan/AGENTS.md b/worker_plan/AGENTS.md index 6892417cd..8e19c30eb 100644 --- a/worker_plan/AGENTS.md +++ b/worker_plan/AGENTS.md @@ -50,6 +50,29 @@ consumers. `rate_limit`, etc.) stored in `usage_metrics.jsonl`. Unknown errors preserve a truncated `error_detail` field. +## Pipeline Stages (`worker_plan_internal/plan/stages/`) + +Each Luigi pipeline task lives in its own file under `stages/`. This enables: +- Multiple agents working on different stages without merge conflicts +- `self_improve/` targeting individual step files +- Easy DAG insertion (create new file, update downstream `requires()`) + +### Convention for new stages + +1. Create `stages/.py` with one task class +2. Import `PlanTask` from `worker_plan_internal.plan.run_plan_pipeline` +3. Import upstream task dependencies from sibling stage files +4. Declare dependencies via `requires()` returning upstream task(s) +5. Add the new task to `stages/full_plan_pipeline.py`'s `requires()` dict + +### Framework location + +`run_plan_pipeline.py` contains only shared framework: +- `PlanTask` (base class for all stages) +- `ExecutePipeline`, `HandleTaskCompletionParameters`, `PipelineProgress` +- `_task_class_to_step_label`, `configure_logging` +- `__main__` entry point + ## Testing - Prefer unit tests over manual server checks. Run `python test.py` from repo root; worker tests live under `worker_plan/worker_plan_internal/**/tests` and From 75e5f29b79e988900a3874f99e2bc17dab0de229 Mon Sep 17 00:00:00 2001 From: Simon Strandgaard Date: Thu, 2 Apr 2026 19:26:05 +0200 Subject: [PATCH 12/12] cleanup: remove unused date import from run_plan_pipeline.py Co-Authored-By: Claude Opus 4.6 (1M context) --- worker_plan/worker_plan_internal/plan/run_plan_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py b/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py index 8f906a9e9..93861faf8 100644 --- a/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py +++ b/worker_plan/worker_plan_internal/plan/run_plan_pipeline.py @@ -7,7 +7,7 @@ PROMPT> RUN_ID_DIR=/absolute/path/to/PlanExe_20250216_150332 python -m worker_plan_internal.plan.run_plan_pipeline """ from dataclasses import dataclass, field -from datetime import date, datetime +from datetime import datetime import os import logging import json