Move include/exclude files lists to projects (CODE_DIR/.autopilot/.env)

[hatkyinc2]

Background story explaining the current situation:
We currently have the environment variables:

FILE_EXTENSIONS_TO_PROCESS=.js,.tsx,.ts,.jsx
IGNORE_LIST=node_modules,coverage,public,__tests__

Inside the .env file that sits inside the autopilot code directory

Desired new situation:
The above variables should be read from the target code directory (CODE_DIR/.autopilot/.env)

Implementation details:
Move include/exclude files lists to projects (CODE_DIR/.autopilot/.env)
Bootstrap such a file under our init function.
Read said file, including other environment variables.
Validate these env variables are set.

• Update the .env.template accordingly
• Update the README accordingly

re: #86
re: #76
re: #163

[codeautopilot]

🤖 This is my suggestion:

File: README.md
To implement the desired changes, follow these steps:

1. Update the "Installation" section in the README.md file to include instructions for setting up the new environment variables in the target code directory (CODE_DIR/.autopilot/.env). Mention that the user should create a .env file inside the .autopilot folder in their code directory and add the following variables:

FILE_EXTENSIONS_TO_PROCESS=.js,.tsx,.ts,.jsx
IGNORE_LIST=node_modules,coverage,public,__tests__

2. In the "Running" section, add a note that the environment variables in the target code directory will be read and used by Autopilot.

3. Update any other relevant sections in the README.md file to reflect the changes made in the codebase.

File: modules/fsInput.js
To update the fsInput.js file to read the new environment variables from the target code directory, follow these steps:

1. Update the require('dotenv').config(); line to accept a path to the target code directory's .env file. You can do this by passing an object with a path property to the config() function, like so:

const targetCodeDir = process.env.CODE_DIR;
require('dotenv').config({ path: path.join(targetCodeDir, '.autopilot', '.env') });

Make sure to add const path = require('path'); at the beginning of the file if it's not already there.

2. Replace the ignoreList and fileExtensionsToProcess variables with the new environment variables:

const ignoreList = process.env.IGNORE_LIST.split(',');
const fileExtensionsToProcess = process.env.FILE_EXTENSIONS_TO_PROCESS.split(',');

After making these changes, the fsInput.js file should now read the new environment variables from the target code directory's .env file.

File: .env.template
Update the .env.template file to remove the FILE_EXTENSIONS_TO_PROCESS and IGNORE_LIST variables, as they will now be read from the target code directory (CODE_DIR/.autopilot/.env).

.env.template

OPENAI_API_KEY=<your-api-key>
# Path to your code, defaults to itself.
# Even if you are on windows this is POSIX style path.
# For example C:\Users\user\code\project
#    would be C:/Users/user/code/project
CODE_DIR=./

# Currently all the models support either 'gpt-3.5-turbo' or 'gpt-4' (if you have access to it)
CODER_MODEL=gpt-3.5-turbo
CODE_READER_MODEL=gpt-3.5-turbo
GET_FILES_MODEL=gpt-3.5-turbo
INDEXER_MODEL=gpt-3.5-turbo
REVIEWER_MODEL=gpt-3.5-turbo
TASK_COMPLEXITY_MODEL=gpt-3.5-turbo
OPENAI_MAX_TOKEN_REPLY=2000 # Max tokens to return from OpenAI
MODEL_TEMPERATURE=0.01 # range 0-1, 0 being the most conservative, 1 being the most creative
MODEL_PRESENCE_PENALTY=0 # range -2 - 2 Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
MODEL_FREQUENCY_PENALTY=0 # range -2 - 2 Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
MODEL_USER=autopilot # Identify this usage
MAX_TOKEN_COUNT_SINGLE_FILE=1800; # Files above this token size would not be processed
MAX_TOKEN_COUNT_SUMMARIES_CHUNK=1000; # Summaries would be chunked to this max size and looped over

File: modules/init.js
To update the initCodeBase function to create the new .env file in the target code directory, follow these steps:

1. Import the necessary modules at the beginning of the file:

const os = require('os');

2. Add a new function createEnvFile that will create the .env file in the target code directory if it doesn't exist:

function createEnvFile(codeBaseAutopilotDirectory) {
  const envFilePath = path.join(codeBaseAutopilotDirectory, '.env');
  if (!fs.existsSync(envFilePath)) {
    const defaultEnvContent = [
      'FILE_EXTENSIONS_TO_PROCESS=.js,.tsx,.ts,.jsx',
      'IGNORE_LIST=node_modules,coverage,public,__tests__',
    ].join(os.EOL);
    fs.writeFileSync(envFilePath, defaultEnvContent);
  }
}

3. Call the createEnvFile function inside the initCodeBase function, after creating the codeBaseAutopilotDirectory:

async function initCodeBase(codeBaseDirectory, interactive) {
  // ...
  if (!fs.existsSync(codeBaseAutopilotDirectory)) {
    fs.mkdirSync(codeBaseAutopilotDirectory);
  }

  // Create .env file in the target code directory
  createEnvFile(codeBaseAutopilotDirectory);

  // ...
}

After these changes, the modules/init.js file should look like this:

const path = require('path');
const { createDB } = require('./db');
const fs = require('fs');
const os = require('os');
const { codeBaseFullIndex, codeBaseFullIndexInteractive } = require('./codeBase');
const { getCodeBaseAutopilotDirectory } = require('./autopilotConfig');

function createEnvFile(codeBaseAutopilotDirectory) {
  const envFilePath = path.join(codeBaseAutopilotDirectory, '.env');
  if (!fs.existsSync(envFilePath)) {
    const defaultEnvContent = [
      'FILE_EXTENSIONS_TO_PROCESS=.js,.tsx,.ts,.jsx',
      'IGNORE_LIST=node_modules,coverage,public,__tests__',
    ].join(os.EOL);
    fs.writeFileSync(envFilePath, defaultEnvContent);
  }
}

/**
 *
 * @param {string} codeBaseDirectory
 */
async function initCodeBase(codeBaseDirectory, interactive) {
  model = process.env.INDEXER_MODEL;
  // Create directory `__CODEBASE__/.autopilot`
  codeBaseAutopilotDirectory = getCodeBaseAutopilotDirectory(codeBaseDirectory);

  if (!fs.existsSync(codeBaseAutopilotDirectory)) {
    fs.mkdirSync(codeBaseAutopilotDirectory);
  }

  // Create .env file in the target code directory
  createEnvFile(codeBaseAutopilotDirectory);

  // Create config file `__CODEBASE__/.autopilot/config.json`
  // TODO: Refactor include/exclude lists into codebase config file

  const { getDBFilePath } = require('./db');
  // Bootstrap DB
  if (!fs.existsSync(getDBFilePath(codeBaseDirectory))) {
    createDB(codeBaseDirectory);
    // Trigger codeBase indexing
    if (interactive) {
      await codeBaseFullIndexInteractive(codeBaseDirectory, model);
    } else {
      await codeBaseFullIndex(codeBaseDirectory);
    }
  }
}

module.exports = { initCodeBase };