[pull] main from TrafficGuard:main

DOCS.md

@@ -303,7 +303,7 @@ Use descriptive variable and function names instead of commenting on generic nam
 # Example - Minimise unnecessary line count
 
 ## Original
-This code was repeated multiple times through the tests
+This code was repeated multiple times through the tests. Extract it into a helper function.
 ```typescript
 			expect(
     await fsAsync
@@ -325,4 +325,176 @@ async function fileExists(filePath: string): Promise<boolean> {
 it('should test foo', async () => {
     expect(await fileExists(orphanedSummaryFullPath)).to.be.true;
 })
-```
+```
+
+## Original
+
+```typescript
+logger.info(
+			`Indexing pipeline completed. Successfully prepared ${successfullyProcessedAndEmbeddedChunks} chunks from ${
+				codeFiles.length - failedFilesCount.count
+			} files for indexing.`,
+		);
+```
+
+# Updated
+```typescript
+const successCount = codeFiles.length - failedFilesCount.count
+logger.info(`Indexing completed. Prepared ${successfullyProcessedAndEmbeddedChunks} chunks from ${successCount} files.`);
+```
+
+## Notes
+Extracting the successCount calculation into a variable and shortening the wording so the statement can be on a single line reduces the line count from 5 down to 2.
+
+## Example
+## Before
+```typescript
+async indexRepository(dir = './'): Promise<void> {
+    const codeFiles = await readFilesToIndex(dir);
+    logger.info(`Loaded ${codeFiles.length} code files.`);
+
+    if (codeFiles.length === 0) {
+        logger.info('No files to index.');
+        return;
+    }
+    // ...
+}
+```
+Review - Its already logged that 0 files were loaded. The additional logging is redundant
+
+### After
+
+```typescript
+async indexRepository(dir = './'): Promise<void> {
+    const codeFiles = await readFilesToIndex(dir);
+    logger.info(`Loaded ${codeFiles.length} code files.`);
+
+    if (codeFiles.length === 0) return;
+    // ...
+}
+```
+
+# Unnecassary seperate function
+
+## Before
+```typescript
+async importDocuments(documents: google.cloud.discoveryengine.v1beta.IDocument[], file: CodeFile): Promise<void> {
+    if (documents.length === 0) return;
+    await this.ensureDataStoreExists();
+
+    const request: google.cloud.discoveryengine.v1beta.IImportDocumentsRequest = {
+        parent: `${this.dataStorePath}/branches/default_branch`,
+        inlineSource: { documents },
+        reconciliationMode: google.cloud.discoveryengine.v1beta.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+    };
+
+    await this._importDocumentsRequest(request);
+}
+
+@cacheRetry({ retries: 3, backOffMs: 1000 })
+@quotaRetry()
+private async _importDocumentsRequest(request: google.cloud.discoveryengine.v1beta.IImportDocumentsRequest): Promise<void> {
+    const [operation] = await this.documentClient.importDocuments(request);
+    logger.info(`ImportDocuments operation started: ${operation.name}`);
+    await operation.promise(); // wait until the indexing finishes
+}
+```
+
+## After
+```typescript
+@cacheRetry({ retries: 3, backOffMs: 1000 })
+@quotaRetry()
+async importDocuments(documents: google.cloud.discoveryengine.v1beta.IDocument[], file: CodeFile): Promise<void> {
+    if (documents.length === 0) return;
+
+    const request: google.cloud.discoveryengine.v1beta.IImportDocumentsRequest = {
+        parent: `${this.dataStorePath}/branches/default_branch`,
+        inlineSource: { documents },
+        reconciliationMode: google.cloud.discoveryengine.v1beta.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
+    };
+
+    const [operation] = await this.documentClient.importDocuments(request);
+    logger.info(`ImportDocuments operation started: ${operation.name}`);
+    await operation.promise(); // wait until the indexing finishes
+}
+```
+
+## Conversion of database data to domain objects
+
+### Example
+
+#### Before
+
+```typescript
+const querySnapshot = await iterationsColRef.orderBy('__name__').get(); // Order by document ID (iteration number)
+
+const iterations: AutonomousIteration[] = [];
+querySnapshot.forEach((doc) => {
+    const data = doc.data();
+    if (data && typeof data.iteration === 'number') {
+        // Ensure memory is a Record, defaulting to {} if missing or not a valid object.
+        data.memory = data.memory && typeof data.memory === 'object' && !Array.isArray(data.memory) ? data.memory : {};
+
+        // Ensure toolState is a Record, defaulting to {} if missing or not a valid object.
+        data.toolState = data.toolState && typeof data.toolState === 'object' && !Array.isArray(data.toolState) ? data.toolState : {};
+
+        // Ensure optional fields are correctly handled (set to undefined if missing/null)
+        data.error = data.error || undefined;
+        data.agentPlan = data.agentPlan || undefined;
+        data.code = data.code || undefined;
+        data.prompt = data.prompt || undefined;
+        data.functionCalls = data.functionCalls || [];
+        data.functions = data.functions || [];
+        data.createdAt = data.createdAt || Date.now();
+        // expandedUserRequest, observationsReasoning, nextStepDetails might also need default handling if optional
+        data.expandedUserRequest = data.expandedUserRequest || undefined;
+        data.observationsReasoning = data.observationsReasoning || undefined;
+        data.nextStepDetails = data.nextStepDetails || undefined;
+        // Ensure optional fields potentially missing from Firestore are set
+        data.draftCode = data.draftCode || undefined;
+        data.codeReview = data.codeReview || undefined;
+
+        iterations.push(data as AutonomousIteration);
+    } else {
+        logger.warn({ agentId, iterationId: doc.id }, 'Skipping invalid iteration data during load (missing or invalid iteration number)');
+    }
+});
+```
+
+#### After
+
+```typescript
+const iterations: AutonomousIteration[] = [];
+querySnapshot.forEach((doc) => {
+    const data = doc.data();
+
+    if (data && typeof data.iteration === 'number') {
+        const iter: AutonomousIteration = {
+            // Ensure memory is a Record, defaulting to {} if missing or not a valid object.
+            memory: data.memory && typeof data.memory === 'object' && !Array.isArray(data.memory) ? data.memory : {},
+            // Ensure toolState is a Record, defaulting to {} if missing or not a valid object.
+            toolState: data.toolState && typeof data.toolState === 'object' && !Array.isArray(data.toolState) ? data.toolState : {},
+            // Handle optional fields using safe defaults
+            error: data.error || undefined,
+            agentPlan: data.agentPlan || undefined,
+            code: data.code || undefined,
+            prompt: data.prompt || undefined,
+            functionCalls: data.functionCalls || [],
+            functions: data.functions || [],
+            createdAt: data.createdAt || Date.now(),
+            expandedUserRequest: data.expandedUserRequest || undefined,
+            observationsReasoning: data.observationsReasoning || undefined,
+            nextStepDetails: data.nextStepDetails || undefined,
+            draftCode: data.draftCode || undefined,
+            codeReview: data.codeReview || undefined,
+            // Required field
+            iteration: data.iteration
+        };
+        iterations.push(iter);
+    } else {
+        logger.warn({ agentId, iterationId: doc.id },'Skipping invalid iteration data during load (missing or invalid iteration number)');
+    }
+});
+```
+### Notes
+Assign to the typed AutonomousIteration object to ensure type safety.

bin/aider

@@ -7,9 +7,11 @@ export OPENAI_API_KEY=$OPENAI_API_KEY
 export DEEPSEEK_API_KEY=$DEEPSEEK_API_KEY
 export OPENROUTER_API_KEY=$OPENROUTER_API_KEY
 export GEMINI_API_KEY=$GEMINI_API_KEY
+export XAI_API_KEY=$XAI_API_KEY
 
 MODEL=vertex_ai/gemini-2.5-pro
 #MODEL=o3
+#MODEL=xai/grok-4
 EDITOR_MODEL=vertex_ai/gemini-2.5-flash
 WEAK_MODEL=vertex_ai/gemini-2.5-flash
 

bin/configure

@@ -94,10 +94,10 @@ echo "---------------------------------------------------------------------"
 
 
 # --- Database Setup ---
-echo ""
-echo "--- Database Setup ---"
-source ./bin/configure_parts/db_setup || exit 1
-echo "---------------------------------------------------------------------"
+#echo ""
+#echo "--- Database Setup ---"
+#source ./bin/configure_parts/db_setup || exit 1
+#echo "---------------------------------------------------------------------"
 
 
 echo ""

bin/configure_parts/DOCS.md

@@ -21,3 +21,21 @@ Example:
 # ./bin/configure_parts/Dockerfile
 
 This is used from configure_test to manually test the setup script in a fresh environment
+
+## Sourcing vs. Executing – Return/Exit Convention
+
+Every script in `bin/configure_parts` **may** be either
+1. sourced by `./bin/configure` (so control must come back), or  
+2. executed directly for stand-alone testing.
+
+To make this reliable, each part **must** finish with:
+
+```bash
+# If sourced, just return; if executed directly, exit.
+(return 0 2>/dev/null) && return 0 || exit 0
+```
+
+• Use `exit <non-zero>` freely for **fatal errors**;  
+• Never use `exit 0` for normal completion—use the pattern above instead.  
+This keeps the parent `configure` script running while still allowing the
+file to act as a self-contained executable.

bin/configure_parts/app_setup

@@ -1,8 +1,5 @@
 #!/bin/bash
-# Source the helper function
 source ./bin/configure_parts/shell_helpers.sh
-# Application Configuration and Frontend Setup ---------------------------------
-echo "--- Starting Application Configuration and Frontend Setup ---"
 
 # Function to update or add a variable in an .env file (local to this script for now)
 # This function is duplicated in configure for robustness, but kept here for standalone testing/clarity.
@@ -97,4 +94,6 @@ content="export TYPEDAI_HOME=$(pwd)\nexport PATH=\$TYPEDAI_HOME/bin/path:\$PATH"
 update_shell_profiles "$comment" "$content"
 
 echo "Application Configuration and Frontend Setup complete."
-exit 0
+
+# If we were sourced, return; if executed directly, exit.                                                                                                     
+(return 0 2>/dev/null) && return 0 || exit 0

bin/configure_parts/firestore_setup

@@ -101,5 +101,6 @@ create_firestore_index "CodeTask" "field-path=userId,order=ascending" "field-pat
 create_firestore_index "Prompts" "field-path=userId,order=ascending" "field-path=updatedAt,order=descending" "field-path=__name__,order=descending"
 
 echo "Firestore Database setup script completed."
-# Exit with 0 to indicate success of this part
-exit 0
+
+# If this file was sourced, return; if executed directly, exit.
+(return 0 2>/dev/null) && return 0 || exit 0

bin/configure_parts/nodejs_setup

@@ -364,4 +364,7 @@ if [ $? -ne 0 ]; then
 fi
 echo "✅ 'npm install' completed successfully."
 
+# If this file was sourced, return; if executed directly, exit.
+(return 0 2>/dev/null) && return 0 || exit 0
+
 # The rest of the script (Angular setup, etc.) will use the Node.js version set by fnm.

bin/configure_parts/postgres_setup

@@ -48,5 +48,7 @@ echo "You may also need to run database migrations after starting the applicatio
 echo "Refer to the documentation for migration instructions."
 
 echo "PostgreSQL Database setup script completed."
-# Exit with 0 to indicate success of this part (even if Docker failed, user might use external DB)
-exit 0
+# Exit/return with 0 to indicate success of this part (even if Docker failed, user might use external DB)
+# If we were sourced, return; if executed directly, exit.                                                                                                     
+(return 0 2>/dev/null) && return 0 || exit 0
+

bin/configure_parts/python_setup

@@ -383,3 +383,6 @@ else
 fi
 
 echo "Python environment setup with pyenv complete."
+
+# If this file was sourced, return; if executed directly, exit.
+(return 0 2>/dev/null) && return 0 || exit 0

bin/configure_parts/ripgrep_setup

@@ -180,3 +180,6 @@ fi
 
 echo "✅ ripgrep (rg) is now installed and available."
 echo "   rg version: $(rg --version | head -n 1)"
+
+# If this file was sourced, return; if executed directly, exit.
+(return 0 2>/dev/null) && return 0 || exit 0

bin/path/DOCS.md

@@ -0,0 +1 @@
+This folder is intended to be added to the users PATH so the commands are available from any directory.

bin/path/ai

@@ -1,13 +1,19 @@
 #!/bin/bash
 
-# Shortcut to running `npm run <script> <arg>` from the TypedAI installation
+# Shortcut for users to run `npm run <script> <arg>` from the TypedAI installation
 # Usage: `ai <script> <args>`
 
 if [ -z "$TYPEDAI_HOME" ]; then
   echo "Error: TYPEDAI_HOME is not set."
   exit 1
 fi
 
+# If there arg is "b" then run ../find_chrome_tab
+if [ "$1" = "b" ]; then
+  $TYPEDAI_HOME/bin/find_chrome_tab
+  exit 0
+fi
+
 # Check if at least one argument is passed
 if [ $# -lt 1 ]; then
   echo "Usage: ai <npm script> [args]"

bin/path/aid

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# This script is for running a TypedAI script on a target repository (not the TypedAI repo) in a container.
+# This should be run from the target repository, and may modify the target repository depending on the script called.
+
 # Check if TYPEDAI_HOME is set
 if [ -z "$TYPEDAI_HOME" ]; then
   echo "Error: TYPEDAI_HOME is not set."

bin/path/port

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Usage: port <port> [-k]
+# -k will kill the process
+
+# Store the output of lsof so we can extract the PID
+OUTPUT=$(sudo lsof -i:$1)
+KILL=""
+if [ "$2" = "-k" ]; then
+  echo "Killing process"
+  KILL="-k"
+fi
+# The output will be like
+# username@MACHINE cwd % port 8243
+# Password:
+# COMMAND   PID     USER   FD   TYPE             DEVICE SIZE/OFF NODE NAME
+# java    44607 johndoe  138u  IPv6 0xce01389c90ae50c9      0t0  TCP localhost:8243 (LISTEN)
+#
+
+# Extract the PID from the output
+PID=$(echo "$OUTPUT" | awk 'NR==2 {print $2}')
+
+# If the -k option is used, the process will be killed.
+if [ "$KILL" = "-k" ]; then
+  sudo kill -9 "$PID"
+fi

bin/path/taider

@@ -0,0 +1,39 @@
+# Convenience script for running Aider from outside the TypedAI repo
+# `source ./variables/local.env` relative to this script file location
+source $(dirname "$0")/../../variables/local.env
+
+export VERTEXAI_PROJECT=$GCLOUD_PROJECT
+export VERTEXAI_LOCATION=$GCLOUD_REGION
+export OPENAI_API_KEY=$OPENAI_API_KEY
+export DEEPSEEK_API_KEY=$DEEPSEEK_API_KEY
+export OPENROUTER_API_KEY=$OPENROUTER_API_KEY
+export GEMINI_API_KEY=$GEMINI_API_KEY
+
+MODEL=vertex_ai/gemini-2.5-pro
+#MODEL=o3
+EDITOR_MODEL=vertex_ai/gemini-2.5-flash
+WEAK_MODEL=vertex_ai/gemini-2.5-flash
+
+# EDIT_FORMAT="--edit-format whole" # Usefull when editing the search-replace coder files with SEARCH/REPLACE blocks in the code and tests
+
+echo $MODEL $EDITOR_MODEL $VERTEXAI_PROJECT $VERTEXAI_LOCATION
+
+# Ensure we use the Python version specified in the repository
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="${TYPEDAI_HOME:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
+if [[ -f "$ROOT_DIR/.python-version" ]]; then
+	PY_VERSION=$(cat "$ROOT_DIR/.python-version")
+	if command -v pyenv >/dev/null 2>&1; then
+		# Tell pyenv which version to use for this invocation
+		export PYENV_VERSION="$PY_VERSION"
+		# Prefer direct path to avoid shim issues; fall back to `pyenv which`
+		AIDER_BIN="$(pyenv root)/versions/$PY_VERSION/bin/aider"
+		[[ -x "$AIDER_BIN" ]] || AIDER_BIN="$(pyenv which aider 2>/dev/null)"
+		[[ -x "$AIDER_BIN" ]] && AIDER_CMD="$AIDER_BIN"
+	fi
+fi
+: ${AIDER_CMD:=aider}
+
+# npm run test && 
+# cd frontend && npm run test:ci
+"$AIDER_CMD" --model $MODEL  --editor-model $EDITOR_MODEL  --weak-model $WEAK_MODEL --no-auto-accept-architect --test-cmd "npm run build" --auto-test