huggingface leaderboard and github action to update markdown leaderbo…

…ard from json

.github/scripts/generate-leaderboard.py

@@ -0,0 +1,63 @@
+import json
+
+
+def generate_markdown(data):
+    markdown = """<div align="center">\n\n"""
+    markdown += "# 🔥🏅️GenCeption Leaderboard 🏅️🔥\n\n"
+    markdown += """\n\n</div>\n\n"""
+    markdown += "#### GC@3 scores for different models and categories:\n"
+    markdown += "| Model | **Mean** | Exist. | Count | Posi. | Col. | Post. | Cel. | Sce. | Lan. | Art. | Comm. | **Vis Mean** | Code | Num. | Tran. | OCR | **Text Mean** |\n"
+    markdown += (
+        "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"
+    )
+    for model in data["models"]:
+        scores = model["scores"]
+        markdown += f"| [{model['name']}]({model['url']}) "
+        for score_key in [
+            "Mean",
+            "Exist",
+            "Count",
+            "Posi",
+            "Col",
+            "Post",
+            "Cel",
+            "Sce",
+            "Lan",
+            "Art",
+            "Comm",
+            "VisMean",
+            "Code",
+            "Num",
+            "Tran",
+            "OCR",
+            "TextMean",
+        ]:
+            if "Mean" in score_key:
+                markdown += f"| **{scores[score_key]}** "
+            else:
+                markdown += f"| {scores[score_key]} "
+        markdown += "|\n"
+    markdown += """\n\nLegend:
+- Exist.: Existence
+- Count: Count
+- Posi.: Position
+- Col.: Color
+- Post.: Poster
+- Cel.: Celebrity
+- Sce.: Scene
+- Lan.: Landmark
+- Art.: Artwork
+- Com. R.: Commonsense Reasoning
+- Code: Code Reasoning
+- Num.: Numerical Calculation
+- Tran.: Text Translation
+- OCR: OCR"""
+    return markdown
+
+
+if __name__ == "__main__":
+    with open("leaderboard/leaderboard.json", "r") as f:
+        data = json.load(f)
+    markdown_content = generate_markdown(data)
+    with open("leaderboard/Leaderboard.md", "w") as f:
+        f.write(markdown_content)

.github/workflows/update-leaderboard.yaml

@@ -0,0 +1,29 @@
+name: Update Markdown from JSON
+
+on:
+  push:
+    paths:
+      - 'leaderboard/leaderboard.json'
+
+jobs:
+  update-markdown:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyyaml
+      - name: Generate Markdown
+        run: python .github/scripts/generate_leaderboard.py
+      - name: Commit and push if changed
+        run: |
+          git config --global user.email "action@github.com"
+          git config --global user.name "GitHub Action"
+          git add README.md
+          git commit -m "Update README.md from JSON data" -a || echo "No changes to commit"
+          git push

README.md

@@ -22,7 +22,7 @@ We demostrate a 5-iteration GenCeption procedure below run on a seed images to e
 
 
 ## Contribute
-Please **create PR (Pull-Request)** to contribute your results to the [🔥🏅️**Leaderboard**🏅️🔥](https://github.com/EQTPartners/GenCeption/blob/main/Leaderboard.md). Start by creating your virtual environment:
+Please add your model details and results to `leaderboard/leaderboard.json` and **create a PR (Pull-Request)** to contribute your results to the [🔥🏅️**Leaderboard**🏅️🔥](https://github.com/EQTPartners/GenCeption/blob/main/leaderboard/Leaderboard.md). Start by creating your virtual environment:
 
 ```{bash}
 conda create --name genception python=3.10 -y

app.py

@@ -0,0 +1,107 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+from leaderboard.content import (
+    TITLE,
+    BANNER,
+    INTRO,
+    INTRO2,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+)
+import gradio as gr
+import pandas as pd
+import json
+
+
+df = pd.DataFrame()
+
+
+def update_data():
+    global df
+    with open("leaderboard/leaderboard.json", "r") as f:
+        data = json.load(f)
+    df = create_dataframe(data)
+
+
+def filter_columns(df, show_all):
+    if show_all:
+        return df
+    else:
+        mean_columns = [col for col in df.columns if "Mean" in col or col == "Model"]
+        return df[mean_columns]
+
+
+def create_dataframe(data):
+    rows = []
+    for model in data["models"]:
+        name_with_link = f'<a href="{model["url"]}" target="_blank" style="color: blue; text-decoration: underline;">{model["name"]}</a>'
+        row = {"Model": name_with_link}
+        row.update(model["scores"])
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+
+    for col in df.columns:
+        if "Mean" in col:
+            df[col] = df[col].apply(lambda x: f"<strong>{x}</strong>")
+
+    return df
+
+
+def update_display(show_all, df):
+    filtered_df = filter_columns(df, show_all)
+    legend_visibility = gr.update(visible=show_all)
+    return filtered_df, legend_visibility
+
+update_data()
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.HTML(BANNER)
+    gr.Markdown(INTRO, elem_classes="markdown-text")
+    gr.Markdown(INTRO2, elem_classes="markdown-text")
+    show_all_columns = gr.Checkbox(label="Show all datasets", value=True)
+    data_display = gr.Dataframe(df, datatype="markdown")
+
+    legend_accordion = gr.Accordion("Legend:", open=False, visible=True)
+    with legend_accordion:
+        gr.Markdown(
+            """
+        - Exist.: Existence
+        - Count: Count
+        - Posi.: Position
+        - Col.: Color
+        - Post.: Poster
+        - Cel.: Celebrity
+        - Sce.: Scene
+        - Lan.: Landmark
+        - Art.: Artwork
+        - Com. R.: Commonsense Reasoning
+        - Code: Code Reasoning
+        - Num.: Numerical Calculation
+        - Tran.: Text Translation
+        - OCR: OCR
+        """
+        )
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+                lines=10,
+                show_copy_button=True,
+            )
+
+    show_all_columns.change(
+        update_display,
+        inputs=[show_all_columns, gr.State(df)],
+        outputs=[data_display, legend_accordion],
+    )
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(update_data, "cron", hour=0)  # Update data once a day at midnight
+scheduler.start()
+
+demo.queue(default_concurrency_limit=40).launch(share=True)