1.[GO set]Run_LLM_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Query GPT-4 for name and analysis using a toy example\n",
    "\n",
    "#### The new prompt includes a query for confidence score and an example analysis to help the LLM in its task.\n",
    "\n",
    "#### The LLM Score has its own column in the output TSV file.\n",
    "\n",
    "#### The JSON config file is updated to use \"GPT-4_1106-preview\" build."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run in the command line  \n",
    "\n",
    "input_file = 'data/GO_term_analysis/toy_example.csv'\n",
    "config = './jsonFiles/GOLLMrun_config.json'\n",
    "set_index = 'GO'\n",
    "gene_column = 'Genes'\n",
    "gene_sep = ' '\n",
    "start = 0\n",
    "end = 5   \n",
    "out_file = 'data/GO_term_analysis/LLM_processed_toy_example_gpt_4'\n",
    "%run query_llm_for_analysis.py --config $config \\\n",
    "            --initialize \\\n",
    "            --input $input_file \\\n",
    "            --input_sep  ','\\\n",
    "            --set_index $set_index \\\n",
    "            --gene_column $gene_column\\\n",
    "            --gene_sep ' ' \\\n",
    "            --start $start \\\n",
    "            --end $end \\\n",
    "            --output_file $out_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run in Jupyter Notebook "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json \n",
    "from utils.openai_query import openai_chat\n",
    "from utils.prompt_factory import make_user_prompt_with_score\n",
    "from utils.server_model_query import server_model_chat\n",
    "from utils.llm_analysis_utils import process_analysis, save_progress\n",
    "from utils.genai_query import query_genai_model\n",
    "from tqdm import tqdm\n",
    "import constant\n",
    "import openai\n",
    "import os\n",
    "import logging\n",
    "import re\n",
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Default run is using GPT4**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load variables\n",
    "initialize = True # if True, then initialize the input table with llm names, analysis and score to None \n",
    "# Replace with your actual values\n",
    "config_file = './jsonFiles/toyexample.json'  # replace with your actual config file \n",
    "input_file = 'data/GO_term_analysis/toy_example_w_contaminated.csv' # replace with your actual input file\n",
    "input_sep = ','  # replace with the separator\n",
    "set_index = 'GO'  # replace with your column name that you want to set as index or None\n",
    "gene_column = 'Genes'  # replace with your actual column name for the gene list\n",
    "gene_sep = ' '  # replace with your actual separator\n",
    "gene_features = None  # replace with your path to the gene features or None if you don't want to include in the prompt\n",
    "direct = False # if True, then the prompt will be a direct sentence asking for a name and analysis from the gene set, otherwise default or customized prompt\n",
    "out_file = 'data/GO_term_analysis/LLM_processed_toy_example_w_contamination_gpt_4'  # replace with your actual output file name\n",
    "\n",
    "customized_prompt = False # if True, then the prompt will be the custom prompt, if False, then the prompt will use default\n",
    "\n",
    "# load the config file\n",
    "with open(config_file) as json_file:\n",
    "    config = json.load(json_file)\n",
    "\n",
    "if customized_prompt:\n",
    "    # make sure the file exist \n",
    "    if os.path.isfile(config['CUSTOM_PROMPT_FILE']):\n",
    "        with open(config['CUSTOM_PROMPT_FILE'], 'r') as f: # replace with your actual customized prompt file\n",
    "            customized_prompt = f.read()\n",
    "            assert len(customized_prompt) > 1, \"Customized prompt is empty\"\n",
    "    else:\n",
    "        print(\"Customized prompt file does not exist\")\n",
    "        customized_prompt = None\n",
    "else:\n",
    "    customized_prompt = None\n",
    "\n",
    "# Load OpenAI key, context, and model used \n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "context = config['CONTEXT']\n",
    "model = config['MODEL']\n",
    "temperature = config['TEMP']\n",
    "max_tokens = config['MAX_TOKENS']\n",
    "if model.startswith('gpt'):\n",
    "    rate_per_token = config['RATE_PER_TOKEN']\n",
    "    DOLLAR_LIMIT = config['DOLLAR_LIMIT']\n",
    "LOG_FILE = config['LOG_NAME']+'_log.json'\n",
    "\n",
    "SEED = constant.SEED\n",
    "column_prefix = model.split('-')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# handle the logger so it create a new one for each model run\n",
    "def get_logger(filename):\n",
    "    logger = logging.getLogger(filename)\n",
    "    logger.setLevel(logging.INFO)\n",
    "    if not logger.handlers:\n",
    "        file_handler = logging.FileHandler(filename)\n",
    "        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
    "        file_handler.setFormatter(formatter)\n",
    "        logger.addHandler(file_handler)\n",
    "    return logger\n",
    "\n",
    "\n",
    "def main(df):\n",
    "    analysis_dict  = {}\n",
    "\n",
    "    logger = get_logger(f'{out_file}.log')\n",
    "\n",
    "    i = 0 #used for track progress and saving the file\n",
    "    for idx, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "        #only process None rows \n",
    "        if pd.notna(row[f'{column_prefix} Analysis']):\n",
    "            continue\n",
    "        \n",
    "        gene_data = row[gene_column]\n",
    "        # if gene_data is not a string, then skip\n",
    "        if type(gene_data) != str:\n",
    "            \n",
    "            logger.warning(f'Gene set {idx} is not a string, skipping')\n",
    "            continue\n",
    "        genes = gene_data.split(gene_sep)\n",
    "        \n",
    "        if len(genes) >1000:\n",
    "            logger.warning(f'Gene set {idx} is too big, skipping')\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            prompt = make_user_prompt_with_score(genes)\n",
    "            # print(prompt)\n",
    "            finger_print = None\n",
    "            if model.startswith('gpt'):\n",
    "                print(\"Accessing OpenAI API\")\n",
    "                analysis, finger_print = openai_chat(context, prompt, model, temperature, max_tokens, rate_per_token, LOG_FILE, DOLLAR_LIMIT, SEED)\n",
    "            elif model.startswith('gemini'):\n",
    "                print(\"Using Google Gemini API\")\n",
    "                analysis, error_message = query_genai_model(f\"{context}\\n{prompt}\", model, temperature, max_tokens, LOG_FILE) \n",
    "            else:\n",
    "                print(\"Using server model\")\n",
    "                analysis, error_message= server_model_chat(context, prompt, model, temperature, max_tokens,LOG_FILE, SEED)\n",
    "\n",
    "            \n",
    "            if analysis:\n",
    "                # print(analysis)\n",
    "                llm_name, llm_score, llm_analysis = process_analysis(analysis)\n",
    "                # clean up the score and return float\n",
    "                try:\n",
    "                    llm_score_value =  float(re.sub(\"[^0-9.-]\", \"\", llm_score))\n",
    "                except ValueError:\n",
    "                    llm_score_value = llm_score\n",
    "            \n",
    "                \n",
    "                df.loc[idx, f'{column_prefix} Name'] = llm_name\n",
    "                df.loc[idx, f'{column_prefix} Analysis'] = llm_analysis\n",
    "                df.loc[idx, f'{column_prefix} Score'] = llm_score_value\n",
    "                analysis_dict[f'{idx}_{column_prefix}'] = analysis\n",
    "                # Log success with fingerprint\n",
    "                logger.info(f'Success for {idx} {column_prefix}.')\n",
    "                if finger_print:\n",
    "                    logger.info(f'GPT_Fingerprint for {idx}: {finger_print}')\n",
    "                    \n",
    "            else:\n",
    "                logger.error(f'Error for query gene set {idx}: {error_message}')\n",
    "\n",
    "        except Exception as e:\n",
    "            logger.error(f'Error for {idx}: {e}')\n",
    "            continue\n",
    "        i += 1\n",
    "        if i % 10 == 0:\n",
    "            save_progress(df, analysis_dict, out_file)\n",
    "            # df.to_csv(f'{out_file}.tsv', sep='\\t', index=True)\n",
    "            print(f\"Saved progress for {i} genesets\")\n",
    "    # save the final file\n",
    "    save_progress(df, analysis_dict, out_file)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define your own loop for running the pipeline\n",
    "## 12-18-2023: this loop is for run the default gene set and the contaminated gene sets \n",
    "## can modify this loop for different models or only run on default gene set\n",
    "\n",
    "##12-27-23: edited the prompt \n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    df = pd.read_csv(input_file, sep=input_sep, index_col=set_index)\n",
    "    \n",
    "    if 'gpt' in model:\n",
    "        name_fix = '_'.join(model.split('-')[:2])\n",
    "    else:\n",
    "        name_fix = model.replace(':', '_')\n",
    "    column_prefix = name_fix + '_default'\n",
    "    print(column_prefix)\n",
    "    \n",
    "    if initialize:\n",
    "        # initialize the input file with llm names, analysis and score to None\n",
    "        df[f'{column_prefix} Name'] = None\n",
    "        df[f'{column_prefix} Analysis'] = None\n",
    "        df[f'{column_prefix} Score'] = None\n",
    "    main(df)  ## run with the real set \n",
    "    \n",
    "    ## run the pipeline for contaiminated gene sets \n",
    "    contaminated_columns = [col for col in df.columns if col.endswith('contaminated_Genes')]\n",
    "    # print(contaminated_columns)\n",
    "    for col in contaminated_columns:\n",
    "        gene_column = col ## Note need to change the gene_column to the contaminated column\n",
    "        contam_prefix = '_'.join(col.split('_')[0:2])\n",
    "        \n",
    "        column_prefix = name_fix + '_' +contam_prefix\n",
    "        print(column_prefix)\n",
    "\n",
    "        if initialize:\n",
    "            # initialize the input file with llm names, analysis and score to None\n",
    "            df[f'{column_prefix} Name'] = None\n",
    "            df[f'{column_prefix} Analysis'] = None\n",
    "            df[f'{column_prefix} Score'] = None\n",
    "        main(df)\n",
    "    df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Write a critical analysis of the biological processes performed by this system of interacting proteins.\n",
      "Base your analysis on prior knowledge available in your training data.\n",
      "After completing your analysis, propose a brief and detailed name for the most prominent biological process performed by the system.\n",
      "    \n",
      "After completing your analysis, please also assign a confidence score to the process name you selected.\n",
      "This score should follow the name in parentheses and range from 0.00 to 1.00. A score of 0.00 indicates the lowest confidence,\n",
      "while 1.00 reflects the highest confidence. This score helps gauge how accurately the chosen name represents the functions and activities\n",
      "within the system of interacting proteins. When determining your score, consider the proportion of genes in the protein system that participate\n",
      "in the identified biological process. For instance, if you select \"Ribosome biogenesis\" as the process name but only a few genes in the system \n",
      "contribute to this process, the score should be lower compared to a scenario where a majority of the genes are involved in \"Ribosome biogenesis\".\n",
      "     \n",
      "Put your chosen name at the top of the analysis as 'Process: <name>’.\n",
      "    \n",
      "Be concise, do not use unnecessary words.\n",
      "Be factual, do not editorialize.\n",
      "Be specific, avoid overly general statements such as 'the proteins are involved in various cellular processes'.\n",
      "Avoid listing facts about individual proteins. Instead, try to group proteins with similar functions and discuss their interplay, synergistyc or antagonistic effects and functional integration within the system.\n",
      "Also avoid choosing generic process names such as 'Cellular Signaling and Regulation'.\n",
      "If you cannot identify a prominent biological process for the proteins in the system, I want you to communicate this in you analysis and name the process: “System of unrelated proteins”. Provide a score of 0.00 for a \"System of unrelated proteins\".\n",
      "    \n",
      "To help you in your work, I am providing an example system of interacting proteins and the corresponding example analysis output.\n",
      "\n",
      "The example system of interacting proteins is:\n",
      "PDX1, SLC2A2, NKX6-1, GLP1, GCG.\n",
      "\n",
      "The example analysis output is:\n",
      "\n",
      "Process: Pancreatic development and glucose homeostasis (0.96)\n",
      "\n",
      "1. PDX1 is a homeodomain transcription factor involved in the specification of the early pancreatic epithelium and its subsequent differentiation. \n",
      "It activates the transcription of several genes including insulin, somatostatin, glucokinase and glucose transporter type 2. \n",
      "It is essential for maintenance of the normal hormone-producing phenotype in the pancreatic beta-cell. \n",
      "In pancreatic acinar cells, forms a complex with PBX1b and MEIS2b and mediates the activation of the ELA1 enhancer.\n",
      "\n",
      "2. NKX6-1 is also a transcription factor involved in the development of pancreatic beta-cells during the secondary transition. \n",
      "Together with NKX2-2 and IRX3, controls the generation of motor neurons in the neural tube and belongs to the neural progenitor \n",
      "factors induced by Sonic Hedgehog (SHH) signals.\n",
      "\n",
      "3.GCG and GLP1, respectively glucagon and glucagon-like peptide 1, are involved in glucose metabolism and homeostasis. \n",
      "GCG raises blood glucose levels by promoting gluconeogenesis and is the counter regulatory hormone of Insulin. \n",
      "GLP1 is a potent stimulator of Glucose-Induced Insulin Secretion (GSIS). Plays roles in gastric motility and suppresses blood glucagon levels. \n",
      "Promotes growth of the intestinal epithelium and pancreatic islet mass both by islet neogenesis and islet cell proliferation.\n",
      "\n",
      "4. SLC2A2, also known as GLUT2, is a facilitative hexose transporter. In hepatocytes, it mediates bi-directional transport of glucose accross the plasma membranes, \n",
      "while in the pancreatic beta-cell, it is the main transporter responsible for glucose uptake and part of the cell's glucose-sensing mechanism. \n",
      "It is involved in glucose transport in the small intestine and kidney too.\n",
      "\n",
      "To summarize, the genes in this set are involved in the specification, differentiation, growth and functionality of the pancreas, \n",
      "with a particular emphasis on the pancreatic beta-cell. Particularly, the architecture of the pancreatic islet ensures proper glucose sensing \n",
      "and homeostasis via a number of different hormones and receptors that can elicit both synergistic and antagonistic effects in the pancreas itself and other peripheral tissues.\n",
      "    \n",
      "\n",
      "Here are the interacting proteins:\n",
      "\n",
      "Proteins: GNG5, TBX5, ISL1, RBPJ, CTNNB1, NOTCH1, SMAD4, EYA1, BMP10, SOX9, HES1, ENG, MKS1, SIX1, TBX3, HAND2, PIM1, BMPR2.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(input_file, sep=input_sep, index_col=set_index)\n",
    "df = df.loc['GO:2000136', :]\n",
    "\n",
    "genes = df['Genes'].split(' ')\n",
    "print(make_user_prompt_with_score(genes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create parameter file to run the 1000 terms in batch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "(900, 6)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "selected_go = pd.read_csv('data/GO_term_analysis/1000_selected_go_contaminated.csv')\n",
    "# create a new dataframe by removing the 100 sets have been already ran \n",
    "model_compare_df = pd.read_csv('data/GO_term_analysis/model_compare/LLM_processed_model_compare_100set_gpt_4.tsv', sep='\\t', index_col='GO')\n",
    "model_compare_GO = model_compare_df.index.tolist()\n",
    "print(len(model_compare_GO))\n",
    "\n",
    "new = selected_go[~selected_go['GO'].isin(model_compare_GO)]\n",
    "print(new.shape)\n",
    "new.to_csv('data/GO_term_analysis/900_selected_go_contaminated.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 0         --end 50         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_0_50\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 50         --end 100         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_50_100\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 100         --end 150         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_100_150\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 150         --end 200         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_150_200\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 200         --end 250         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_200_250\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 250         --end 300         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_250_300\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 300         --end 350         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_300_350\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 350         --end 400         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_350_400\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 400         --end 450         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_400_450\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 450         --end 500         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_450_500\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 500         --end 550         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_500_550\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 550         --end 600         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_550_600\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 600         --end 650         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_600_650\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 650         --end 700         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_650_700\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 700         --end 750         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_700_750\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 750         --end 800         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_750_800\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 800         --end 850         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_800_850\n",
      "--config ./jsonFiles/thousandGOrunGPT4_config.json         --initialize         --input ./data/GO_term_analysis/900_selected_go_contaminated.csv         --input_sep  ','        --set_index GO         --gene_column Genes        --gene_sep ' '         --start 850         --end 900         --output_file ./data/GO_term_analysis/LLM_processed_gpt_4_850_900\n",
      "number of params:  18\n"
     ]
    }
   ],
   "source": [
    "## set up parameters for running gpt4 pipeline for the 1000 gene sets\n",
    "import os \n",
    "from glob import glob\n",
    "# Define start, step, and end values\n",
    "start = 0\n",
    "step = 50\n",
    "end = 900 #already ran 100 before\n",
    "\n",
    "# Create a range list\n",
    "range_list = list(range(start, end + step, step))\n",
    "\n",
    "# Create tuples for each consecutive pair in the list\n",
    "tuple_list = [(range_list[i], range_list[i+1]) for i in range(len(range_list)-1)]\n",
    "\n",
    "\n",
    "initialize = True \n",
    "input_file = './data/GO_term_analysis/900_selected_go_contaminated.csv'\n",
    "input_sep = constant.GO_FILE_SEP\n",
    "set_index = constant.GO_INDEX_COL  \n",
    "gene_column = constant.GO_GENE_COL \n",
    "gene_sep = ' '\n",
    "\n",
    "## create a param file \n",
    "conf_file = './jsonFiles/thousandGOrunGPT4_config.json'\n",
    "params = []\n",
    "for start, end in tuple_list:\n",
    "    out_file = f'./data/GO_term_analysis/LLM_processed_gpt_4_{start}_{end}'  \n",
    "    param = f\"--config {conf_file} \\\n",
    "        --initialize \\\n",
    "        --input {input_file} \\\n",
    "        --input_sep  '{input_sep}'\\\n",
    "        --set_index {set_index} \\\n",
    "        --gene_column {gene_column}\\\n",
    "        --gene_sep '{gene_sep}' \\\n",
    "        --start {start} \\\n",
    "        --end {end} \\\n",
    "        --output_file {out_file}\"\n",
    "    print(param)\n",
    "    params.append(param)\n",
    "print('number of params: ', len(params))\n",
    "\n",
    "with open('thousandGOsets_GPT4Run_params.txt', 'w') as f:\n",
    "    for p in params:\n",
    "        f.write(p+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For CC and MF branch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 0             --end 50             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_0_50\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 50             --end 100             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_50_100\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 100             --end 150             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_100_150\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 150             --end 200             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_150_200\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 200             --end 250             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_200_250\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 250             --end 300             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_250_300\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 300             --end 350             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_300_350\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 350             --end 400             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_350_400\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 400             --end 450             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_400_450\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 450             --end 500             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_450_500\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 500             --end 550             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_500_550\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 550             --end 600             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_550_600\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 600             --end 650             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_600_650\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 650             --end 700             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_650_700\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 700             --end 750             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_700_750\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 750             --end 800             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_750_800\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 800             --end 850             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_800_850\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 850             --end 900             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_850_900\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 900             --end 950             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_900_950\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/CC_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 950             --end 1000             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_CCterms_gpt_4_950_1000\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 0             --end 50             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_0_50\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 50             --end 100             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_50_100\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 100             --end 150             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_100_150\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 150             --end 200             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_150_200\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 200             --end 250             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_200_250\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 250             --end 300             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_250_300\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 300             --end 350             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_300_350\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 350             --end 400             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_350_400\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 400             --end 450             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_400_450\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 450             --end 500             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_450_500\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 500             --end 550             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_500_550\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 550             --end 600             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_550_600\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 600             --end 650             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_600_650\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 650             --end 700             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_650_700\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 700             --end 750             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_700_750\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 750             --end 800             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_750_800\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 800             --end 850             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_800_850\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 850             --end 900             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_850_900\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 900             --end 950             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_900_950\n",
      "--config ./jsonFiles/thousandGO_CC_MF_runGPT4_config.json             --initialize             --input ./data/GO_term_analysis/CC_MF_branch/MF_1000_selected_go_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 950             --end 1000             --output_file ./data/GO_term_analysis/CC_MF_branch/LLM_processed_MFterms_gpt_4_950_1000\n",
      "number of params:  40\n"
     ]
    }
   ],
   "source": [
    "## set up parameters for running gpt4 pipeline for the 1000 gene sets for CC and MF\n",
    "import os \n",
    "from glob import glob\n",
    "# Define start, step, and end values\n",
    "start = 0\n",
    "step = 50\n",
    "end = 1000 \n",
    "\n",
    "\n",
    "# Create a range list\n",
    "range_list = list(range(start, end + step, step))\n",
    "\n",
    "# Create tuples for each consecutive pair in the list\n",
    "tuple_list = [(range_list[i], range_list[i+1]) for i in range(len(range_list)-1)]\n",
    "\n",
    "input_sep = constant.GO_FILE_SEP\n",
    "set_index = constant.GO_INDEX_COL  \n",
    "gene_column = constant.GO_GENE_COL \n",
    "gene_sep = ' '\n",
    "\n",
    "## create a param file \n",
    "conf_file = './jsonFiles/thousandGO_CC_MF_runGPT4_config.json'\n",
    "params = []\n",
    "for branch in ['CC', 'MF']:\n",
    "    input_file = f'./data/GO_term_analysis/CC_MF_branch/{branch}_1000_selected_go_terms.csv'\n",
    "    for start, end in tuple_list:\n",
    "        out_file = f'./data/GO_term_analysis/CC_MF_branch/LLM_processed_{branch}terms_gpt_4_{start}_{end}'  \n",
    "        param = f\"--config {conf_file} \\\n",
    "            --initialize \\\n",
    "            --input {input_file} \\\n",
    "            --input_sep  '{input_sep}'\\\n",
    "            --set_index {set_index} \\\n",
    "            --gene_column {gene_column}\\\n",
    "            --gene_sep '{gene_sep}' \\\n",
    "            --start {start} \\\n",
    "            --end {end} \\\n",
    "            --output_file {out_file}\"\n",
    "        print(param)\n",
    "        params.append(param)\n",
    "print('number of params: ', len(params))\n",
    "\n",
    "with open('thousandGOsets_CC_MF_GPT4Run_params.txt', 'w') as f:\n",
    "    for p in params:\n",
    "        f.write(p+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checkout and combine the output from the batch run "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "Any duplicated GO:  0\n",
      "(50, 9)\n",
      "(900, 9)\n",
      "Any duplicated GO:  0\n",
      "Any duplicated LLM analysis:  0\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "### sanity check code along the way\n",
    "processed_files = glob('data/GO_term_analysis/LLM_processed_gpt_4*.tsv')\n",
    "\n",
    "for file in processed_files:\n",
    "    df = pd.read_csv(file, sep='\\t')\n",
    "    df.set_index('GO', inplace=True)\n",
    "    # check if the Analysis, Name and Score are all filled\n",
    "    columns = [col for col in df.columns if col.endswith('Analysis') or col.endswith('Name') or col.endswith('Score')]\n",
    "    print(columns)\n",
    "    for col in columns:\n",
    "        n_na = df[col].isna().sum()\n",
    "        if n_na > 0:\n",
    "            print(f'Error in {file} for {col}, has {n_na} NAs')\n",
    "            print(df[df[col].isna()])\n",
    "        else:\n",
    "            continue\n",
    "    # check if there is any duplicated GO terms\n",
    "    print('Any duplicated GO: ',df.index.duplicated().sum())\n",
    "    \n",
    "    df.reset_index(inplace=True)\n",
    "#     # print(ranges)\n",
    "    print(df.shape)\n",
    "\n",
    "    \n",
    "combined_df = pd.concat([pd.read_csv(f, sep = '\\t') for f in processed_files])\n",
    "print(combined_df.shape)\n",
    "print('Any duplicated GO: ',combined_df['GO'].duplicated().sum())\n",
    "analysis_columns = [col for col in combined_df.columns if col.endswith('Analysis')]\n",
    "print('Any duplicated LLM analysis: ', combined_df[analysis_columns[0]].duplicated(keep=False).sum())\n",
    "\n",
    "combined_df.to_csv('data/GO_term_analysis/LLM_processed_selected_1000_go_terms.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['GO', 'Genes', 'Gene_Count', 'Term_Description', '50perc_contaminated_Genes', '100perc_contaminated_Genes', 'gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(100, 9)\n",
      "(1000, 9)\n",
      "Any duplicated GO:  0\n"
     ]
    }
   ],
   "source": [
    "#combine with the 100 sets that are already ran \n",
    "model_compare_df = pd.read_csv('data/GO_term_analysis/model_compare/LLM_processed_model_compare_100set_gpt_4.tsv', sep='\\t')\n",
    "common_cols = [col for col in model_compare_df.columns if col in combined_df.columns]\n",
    "print(common_cols)\n",
    "model_compare_df = model_compare_df.loc[:, common_cols]\n",
    "print(model_compare_df.shape)\n",
    "\n",
    "combined_df = pd.concat([combined_df, model_compare_df])\n",
    "print(combined_df.shape)\n",
    "print('Any duplicated GO: ',combined_df['GO'].duplicated().sum())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "combined_df.to_csv('data/GO_term_analysis/LLM_processed_selected_1000_go_terms.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test the cost and time usage \n",
    "import json \n",
    "from glob import glob\n",
    "\n",
    "logs = glob('./logs/thousand_GO_run_gpt4*.log')\n",
    "total_cost = 0\n",
    "total_run = 0\n",
    "time_total = 0\n",
    "for log in logs:\n",
    "    with open(log, 'r') as f:\n",
    "        data = json.load(f)\n",
    "        total_cost += data['dollars_spent']\n",
    "        total_run += data['runs']\n",
    "        time_total += data['time_taken_total']\n",
    "print('total cost: {:.2f}'.format(total_cost))\n",
    "print('cost per run: {:.2f}'.format(total_cost/total_run))\n",
    "print('time per run: {:.2f}'.format(time_total/total_run))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine the 1000 terms for MF and CC branch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "(1000, 7)\n",
      "Any duplicated GO in CC combined file:  0\n",
      "Any duplicated CC LLM analysis:  0\n",
      "20\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "['gpt_4_default Name', 'gpt_4_default Analysis', 'gpt_4_default Score']\n",
      "(50, 7)\n",
      "(1000, 7)\n",
      "Any duplicated GO in MF combined file:  0\n",
      "Any duplicated MF LLM analysis:  0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "### sanity check code along the way\n",
    "\n",
    "branches = ['CC', 'MF']\n",
    "for branch in branches:\n",
    "    branch_processed_files = glob(f'data/GO_term_analysis/CC_MF_branch/LLM_processed_{branch}terms_gpt_4*.tsv')\n",
    "    print(len(branch_processed_files))\n",
    "    for file in branch_processed_files:\n",
    "        df = pd.read_csv(file, sep='\\t')\n",
    "        df.set_index('GO', inplace=True)\n",
    "        # check if the Analysis, Name and Score are all filled\n",
    "        columns = [col for col in df.columns if col.endswith('Analysis') or col.endswith('Name') or col.endswith('Score')]\n",
    "        print(columns)\n",
    "        for col in columns:\n",
    "            n_na = df[col].isna().sum()\n",
    "            if n_na > 0:\n",
    "                print(f'Error in {file} for {col}, has {n_na} NAs')\n",
    "                print(df[df[col].isna()])\n",
    "            else:\n",
    "                continue\n",
    "        # check if there is any duplicated GO terms\n",
    "        if df.index.duplicated().sum() > 0:\n",
    "            print('Number of duplicated GO: ',df.index.duplicated().sum())\n",
    "        \n",
    "        df.reset_index(inplace=True)\n",
    "    #     # print(ranges)\n",
    "        print(df.shape)\n",
    "\n",
    "        \n",
    "    combined_df = pd.concat([pd.read_csv(f, sep = '\\t') for f in branch_processed_files])\n",
    "    print(combined_df.shape)\n",
    "    print(f'Any duplicated GO in {branch} combined file: ',combined_df['GO'].duplicated().sum())\n",
    "    analysis_columns = [col for col in combined_df.columns if col.endswith('Analysis')]\n",
    "    print(f'Any duplicated {branch} LLM analysis: ', combined_df[analysis_columns[0]].duplicated(keep=False).sum())\n",
    "\n",
    "    combined_df.to_csv(f'data/GO_term_analysis/CC_MF_branch/LLM_processed_selected_1000_go_{branch}terms.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}