notebooks/inpainting_consistency_table.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "%load_ext autoreload\n",
                "%autoreload 2\n",
                "\n",
                "import json\n",
                "from pathlib import Path\n",
                "from nerfiller.experiments.data import occluder_dataset_names"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def get_metrics(datasets, method_name, dataset_inpaint_method, metrics_list):\n",
                "    \"\"\"Given a list of datasets, compute the average metrics for each dataset.\"\"\"\n",
                "    metrics = [[] for _ in metrics_list]\n",
                "    for dataset in datasets:\n",
                "        # print(dataset, method_name)\n",
                "        filename = str(sorted(list(Path(\"../outputs/\" + dataset + \"-\" + dataset_inpaint_method + \"/\" + method_name).iterdir()))[-1]) + \"/final-renders/test-metrics/test.json\"\n",
                "        # print(filename)\n",
                "        with open(filename, \"r\") as f:\n",
                "            metrics_dict = json.load(f)\n",
                "            for idx, metric_name in enumerate(metrics_list):\n",
                "                metrics[idx].append(metrics_dict[metric_name])\n",
                "    # get the mean\n",
                "    metrics = [sum(metric) / len(metric) for metric in metrics]\n",
                "    return metrics"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "TEMPLATE = \\\n",
                "\"\"\"\n",
                "\\\\begin{table}[]\n",
                "    \\\\captionsetup{font=scriptsize}\n",
                "    \\\\scriptsize\n",
                "    \\\\small\n",
                "    \\\\begin{tabular}{l|lll} % the number of columns needed\n",
                "        \\\\toprule\n",
                "        COLUMNS \\\\\\\\\n",
                "        \\\\midrule\n",
                "        EXPERIMENT_LINES\n",
                "        \\\\bottomrule\n",
                "    \\\\end{tabular}\n",
                "    \\\\caption{\\\\textbf{CAPTION_TITLE.} CAPTION_TEXT}\n",
                "    \\\\label{tab:TABLE_LABEL}\n",
                "\\\\end{table}\n",
                "\"\"\"\n",
                "\n",
                "# TEMPLATE = \\\n",
                "# \"\"\"\n",
                "# \\\\begin{table}[]\n",
                "#     \\\\captionsetup{font=scriptsize}\n",
                "#     \\\\scriptsize\n",
                "#     \\\\footnotesize\n",
                "#     \\\\begin{tabular}{l|lll|ll} % the number of columns needed\n",
                "#         \\\\toprule\n",
                "#         COLUMNS \\\\\\\\\n",
                "#         \\\\midrule\n",
                "#         EXPERIMENT_LINES\n",
                "#         \\\\bottomrule\n",
                "#     \\\\end{tabular}\n",
                "#     \\\\caption{\\\\textbf{CAPTION_TITLE.} CAPTION_TEXT}\n",
                "#     \\\\label{tab:TABLE_LABEL}\n",
                "# \\\\end{table}\n",
                "# \"\"\""
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# # SYNTHETIC EXPERIMENTS\n",
                "# methods are the columns\n",
                "# here we map the title in the column to the the method name (which we can grab from folder names)\n",
                "# COLUMNS = \"\\n & PSNR $\\\\uparrow$ & SSIM $\\\\uparrow$ & LPIPS $\\\\downarrow$\"\n",
                "# METRICS_LIST = [\"psnr\", \"ssim\", \"lpips\"]\n",
                "# SIGFIGS = [2, 2, 2]\n",
                "# assert len(METRICS_LIST) == len(SIGFIGS)\n",
                "# METHODS = [\n",
                "#     (\"nerfacto-nerfiller\", \"none\", \"Masked NeRF\"),\n",
                "#     (\"nerfacto-nerfiller\", \"individual-lama\", \"LaMask\"),\n",
                "#     (\"nerfacto-nerfiller\", \"individual-sd-text\", \"SD Text Cond\"),\n",
                "#     (\"nerfacto-nerfiller\", \"individual-sd-image\", \"SD Image Cond\"),\n",
                "#     (\"nerfacto-nerfiller\", \"expanded-attention\", \"Extended Attention []\"),\n",
                "#     (\"nerfacto-nerfiller\", \"grid-prior-no-joint\", \"Grid Prior\"),\n",
                "#     (\"nerfacto-nerfiller\", \"grid-prior\", \"Multi-Grid Prior\"),\n",
                "# ]\n",
                "# DATASETS = [\"chair\", \"drums\", \"ficus\", \"hotdog\", \"lego\", \"materials\", \"mic\", \"ship\"]\n",
                "# TABLE_LABEL = \"2d_inpainting_consistency\"\n",
                "# caption_title = \"2D inpainting consistency\"\n",
                "# caption_text = \"\"\"\n",
                "# We use various inpainting methods to inpaint images once and then train a NeRF.\n",
                "# These are the average metrics reported over all 8 scenes of the NeRF synthetic dataset.\n",
                "# Better metrics indicate more consistency of the 3D reconstruction with the 2D inpaints.\n",
                "# The Grid Prior method achieves the most consistent results. Please see supplementary material for video rendering results.\n",
                "# \"\"\"\n",
                "\n",
                "# 3D OCCLUDER EXPERIMENTS\n",
                "# methods are the columns\n",
                "# here we map the title in the column to the the method name (which we can grab from folder names)\n",
                "COLUMNS = \"\\n & PSNR $\\\\uparrow$ & SSIM $\\\\uparrow$ & LPIPS $\\\\downarrow$ & MUSIQ $\\\\uparrow$ & Corrs $\\\\uparrow$\"\n",
                "METRICS_LIST = [\"psnr\", \"ssim\", \"lpips\", \"musiq\", \"corrs\"]\n",
                "SIGFIGS = [2, 2, 2, 2, 0]\n",
                "assert len(METRICS_LIST) == len(SIGFIGS)\n",
                "METHODS = [\n",
                "    (\"nerfacto-nerfiller\", \"none\", \"Masked NeRF\"),\n",
                "    (\"individual-inpaint-once\", \"individual-lama\", \"LaMask\"),\n",
                "    (\"individual-inpaint-once\", \"individual-sd-image\", \"SD Image Cond\"),\n",
                "    (\"individual-inpaint-du\", \"none\", \"Inpaint + DU\"),\n",
                "    (\"grid-prior-du-no-depth\", \"none\", \"Ours w/o depth\"),\n",
                "    (\"grid-prior-du\", \"none\", \"Ours\"),\n",
                "]\n",
                "DATASETS = list(occluder_dataset_names)\n",
                "TABLE_LABEL = \"quantitative_nerf_baselines\"\n",
                "caption_title = \"Quantitative NeRF baselines\"\n",
                "caption_text = \"\"\"\n",
                "Here we compare with various methods implemented within the Nerfstudio framework.\n",
                "We report metrics computed in the unknown mask region to quantify consistency of the 3D inpainted region.\n",
                "\"\"\"\n",
                "\n",
                "def print_datasets(datasets, caption_title=\"caption title\", TABLE_LABEL=\"label\", caption_text=\"caption text\"):\n",
                "    # metrics = get_metrics(datasets, \"nerfacto-visibility-sparsity\", METRICS_LIST)\n",
                "    # metrics = [str(f\"{metric:.{sigfigs}f}\") for metric, sigfigs in zip(metrics, SIGFIGS)]\n",
                "\n",
                "    # create the experiment lines\n",
                "    EXPERIMENT_LINES = [\"\\n\"]\n",
                "    for idx, (method_name, dataset_inpaint_method, table_name) in enumerate(METHODS):\n",
                "        # if idx == 1:\n",
                "        #     line_string = f\"\\\\midrule \\n\"\n",
                "        #     EXPERIMENT_LINES.append(line_string)\n",
                "        metrics = get_metrics(datasets, method_name, dataset_inpaint_method, METRICS_LIST)\n",
                "        metrics = [str(f\"{metric:.{sigfigs}f}\") for metric, sigfigs in zip(metrics, SIGFIGS)]\n",
                "        line_string = f\"{table_name} & {' & '.join(metrics)} \\\\\\\\ \\n\"\n",
                "        EXPERIMENT_LINES.append(line_string)\n",
                "\n",
                "    s = TEMPLATE.replace(\"COLUMNS\", COLUMNS)\n",
                "    s = s.replace(\"EXPERIMENT_LINES\", \" \".join(EXPERIMENT_LINES))\n",
                "    s = s.replace(\"CAPTION_TITLE\", caption_title)\n",
                "    s = s.replace(\"CAPTION_TEXT\", caption_text)\n",
                "    s = s.replace(\"TABLE_LABEL\", TABLE_LABEL)\n",
                "    print(s)\n",
                "\n",
                "\n",
                "# for the average of all the datasets\n",
                "print_datasets(DATASETS, caption_title=caption_title, TABLE_LABEL=TABLE_LABEL, caption_text=caption_text)\n",
                "\n",
                "# print(\"---\" * 10)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# for all the datasets individually\n",
                "for i in range(len(DATASETS)):\n",
                "    caption_title_indiv = f\"{caption_title} for data ``{DATASETS[i]}\\\"\"\n",
                "    print_datasets(DATASETS[i : i + 1], caption_title=caption_title_indiv, TABLE_LABEL=DATASETS[i], caption_text=\"\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "nerfiller",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.8.16"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}