Queue concurrency count (#6172)

* changes

* add changeset

* changes

* changes

* changes

* Update client/python/test/conftest.py

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

* changes

* changes

* changes

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/shaky-feet-boil.md

@@ -0,0 +1,5 @@
+---
+"gradio": minor
+---
+
+feat:Queue concurrency count

client/python/test/conftest.py

@@ -38,7 +38,7 @@ def calculator(num1, operation, num2):
             [0, "subtract", 1.2],
         ],
     )
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -48,7 +48,7 @@ def state_demo():
         ["textbox", "state"],
         ["textbox", "state"],
     )
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -81,7 +81,7 @@ def increment_demo():
             api_name=False,
         )
 
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -92,7 +92,7 @@ def my_function(x, progress=gr.Progress()):
             time.sleep(0.1)
         return x
 
-    return gr.Interface(my_function, gr.Textbox(), gr.Textbox()).queue()
+    return gr.Interface(my_function, gr.Textbox(), gr.Textbox())
 
 
 @pytest.fixture
@@ -126,7 +126,7 @@ def long_process():
         btn2 = gr.Button(value="Long Process")
         btn2.click(long_process, None, num, api_name="long")
 
-    return demo.queue(concurrency_count=40)
+    return demo
 
 
 @pytest.fixture
@@ -177,7 +177,7 @@ def show(n):
         count_btn.click(count, num, out)
         list_btn.click(show, num, out)
 
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -202,7 +202,7 @@ def show(n):
         count_btn.click(count, num, out, api_name=False)
         list_btn.click(show, num, out, api_name=False)
 
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -228,7 +228,7 @@ def show(n):
 
         count_btn.click(count, num, out, api_name="count")
         count_forever.click(show, num, out, api_name="count_forever", every=3)
-    return demo.queue()
+    return demo
 
 
 @pytest.fixture
@@ -258,7 +258,6 @@ def respond(message, st, chat_history):
 
         msg.submit(respond, [msg, st, chatbot], [msg, chatbot], api_name="submit")
         clear.click(lambda: None, None, chatbot, queue=False)
-        demo.queue()
     return demo
 
 
@@ -346,7 +345,7 @@ def _stream_audio(audio_file):
         fn=_stream_audio,
         inputs=gr.Audio(type="filepath", label="Audio file to stream"),
         outputs=gr.Audio(autoplay=True, streaming=True),
-    ).queue()
+    )
 
 
 @pytest.fixture

demo/cancel_events/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: cancel_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import time\n", "import gradio as gr\n", "\n", "\n", "def fake_diffusion(steps):\n", "    for i in range(steps):\n", "        print(f\"Current step: {i}\")\n", "        time.sleep(1)\n", "        yield str(i)\n", "\n", "\n", "def long_prediction(*args, **kwargs):\n", "    time.sleep(10)\n", "    return 42\n", "\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            n = gr.Slider(1, 10, value=9, step=1, label=\"Number Steps\")\n", "            run = gr.Button(value=\"Start Iterating\")\n", "            output = gr.Textbox(label=\"Iterative Output\")\n", "            stop = gr.Button(value=\"Stop Iterating\")\n", "        with gr.Column():\n", "            textbox = gr.Textbox(label=\"Prompt\")\n", "            prediction = gr.Number(label=\"Expensive Calculation\")\n", "            run_pred = gr.Button(value=\"Run Expensive Calculation\")\n", "        with gr.Column():\n", "            cancel_on_change = gr.Textbox(label=\"Cancel Iteration and Expensive Calculation on Change\")\n", "            cancel_on_submit = gr.Textbox(label=\"Cancel Iteration and Expensive Calculation on Submit\")\n", "            echo = gr.Textbox(label=\"Echo\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            image = gr.Image(sources=[\"webcam\"], tool=\"editor\", label=\"Cancel on edit\", interactive=True)\n", "        with gr.Column():\n", "            video = gr.Video(sources=[\"webcam\"], label=\"Cancel on play\", interactive=True)\n", "\n", "    click_event = run.click(fake_diffusion, n, output)\n", "    stop.click(fn=None, inputs=None, outputs=None, cancels=[click_event])\n", "    pred_event = run_pred.click(fn=long_prediction, inputs=[textbox], outputs=prediction)\n", "\n", "    cancel_on_change.change(None, None, None, cancels=[click_event, pred_event])\n", "    cancel_on_submit.submit(lambda s: s, cancel_on_submit, echo, cancels=[click_event, pred_event])\n", "    image.edit(None, None, None, cancels=[click_event, pred_event])\n", "    video.play(None, None, None, cancels=[click_event, pred_event])\n", "\n", "    demo.queue(concurrency_count=2, max_size=20)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: cancel_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import time\n", "import gradio as gr\n", "\n", "\n", "def fake_diffusion(steps):\n", "    for i in range(steps):\n", "        print(f\"Current step: {i}\")\n", "        time.sleep(1)\n", "        yield str(i)\n", "\n", "\n", "def long_prediction(*args, **kwargs):\n", "    time.sleep(10)\n", "    return 42\n", "\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            n = gr.Slider(1, 10, value=9, step=1, label=\"Number Steps\")\n", "            run = gr.Button(value=\"Start Iterating\")\n", "            output = gr.Textbox(label=\"Iterative Output\")\n", "            stop = gr.Button(value=\"Stop Iterating\")\n", "        with gr.Column():\n", "            textbox = gr.Textbox(label=\"Prompt\")\n", "            prediction = gr.Number(label=\"Expensive Calculation\")\n", "            run_pred = gr.Button(value=\"Run Expensive Calculation\")\n", "        with gr.Column():\n", "            cancel_on_change = gr.Textbox(label=\"Cancel Iteration and Expensive Calculation on Change\")\n", "            cancel_on_submit = gr.Textbox(label=\"Cancel Iteration and Expensive Calculation on Submit\")\n", "            echo = gr.Textbox(label=\"Echo\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            image = gr.Image(sources=[\"webcam\"], tool=\"editor\", label=\"Cancel on edit\", interactive=True)\n", "        with gr.Column():\n", "            video = gr.Video(sources=[\"webcam\"], label=\"Cancel on play\", interactive=True)\n", "\n", "    click_event = run.click(fake_diffusion, n, output)\n", "    stop.click(fn=None, inputs=None, outputs=None, cancels=[click_event])\n", "    pred_event = run_pred.click(fn=long_prediction, inputs=[textbox], outputs=prediction)\n", "\n", "    cancel_on_change.change(None, None, None, cancels=[click_event, pred_event])\n", "    cancel_on_submit.submit(lambda s: s, cancel_on_submit, echo, cancels=[click_event, pred_event])\n", "    image.edit(None, None, None, cancels=[click_event, pred_event])\n", "    video.play(None, None, None, cancels=[click_event, pred_event])\n", "\n", "    demo.queue(max_size=20)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/cancel_events/run.py

@@ -44,7 +44,7 @@ def long_prediction(*args, **kwargs):
     image.edit(None, None, None, cancels=[click_event, pred_event])
     video.play(None, None, None, cancels=[click_event, pred_event])
 
-    demo.queue(concurrency_count=2, max_size=20)
+    demo.queue(max_size=20)
 
 if __name__ == "__main__":
     demo.launch()

demo/concurrency_with_queue/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: concurrency_with_queue"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import time\n", "\n", "\n", "def say_hello(name):\n", "  time.sleep(5)\n", "  return f\"Hello {name}!\"\n", "\n", "\n", "with gr.Blocks() as demo:\n", "  inp = gr.Textbox()\n", "  outp = gr.Textbox()\n", "  button = gr.Button()\n", "  button.click(say_hello, inp, outp)\n", "\n", "  demo.queue(concurrency_count=41).launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: concurrency_with_queue"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import time\n", "\n", "\n", "def say_hello(name):\n", "  time.sleep(5)\n", "  return f\"Hello {name}!\"\n", "\n", "\n", "with gr.Blocks() as demo:\n", "  inp = gr.Textbox()\n", "  outp = gr.Textbox()\n", "  button = gr.Button()\n", "  button.click(say_hello, inp, outp)\n", "\n", "  demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/concurrency_with_queue/run.py

@@ -13,4 +13,4 @@ def say_hello(name):
   button = gr.Button()
   button.click(say_hello, inp, outp)
 
-  demo.queue(concurrency_count=41).launch()
+  demo.launch()

demo/progress/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: progress"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio tqdm datasets"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import random\n", "import time\n", "import tqdm\n", "from datasets import load_dataset\n", "import shutil\n", "from uuid import uuid4\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        text = gr.Textbox()\n", "        textb = gr.Textbox()\n", "    with gr.Row():\n", "        load_set_btn = gr.Button(\"Load Set\")\n", "        load_nested_set_btn = gr.Button(\"Load Nested Set\")\n", "        load_random_btn = gr.Button(\"Load Random\")\n", "        clean_imgs_btn = gr.Button(\"Clean Images\")\n", "        wait_btn = gr.Button(\"Wait\")\n", "        do_all_btn = gr.Button(\"Do All\")\n", "        track_tqdm_btn = gr.Button(\"Bind TQDM\")\n", "        bind_internal_tqdm_btn = gr.Button(\"Bind Internal TQDM\")\n", "\n", "    text2 = gr.Textbox()\n", "\n", "    # track list\n", "    def load_set(text, text2, progress=gr.Progress()):\n", "        imgs = [None] * 24\n", "        for img in progress.tqdm(imgs, desc=\"Loading from list\"):\n", "            time.sleep(0.1)\n", "        return \"done\"\n", "    load_set_btn.click(load_set, [text, textb], text2)\n", "\n", "    # track nested list\n", "    def load_nested_set(text, text2, progress=gr.Progress()):\n", "        imgs = [[None] * 8] * 3\n", "        for img_set in progress.tqdm(imgs, desc=\"Nested list\"):\n", "            time.sleep(2)\n", "            for img in progress.tqdm(img_set, desc=\"inner list\"):\n", "                time.sleep(0.1)\n", "        return \"done\"\n", "    load_nested_set_btn.click(load_nested_set, [text, textb], text2)\n", "\n", "    # track iterable of unknown length\n", "    def load_random(data, progress=gr.Progress()):\n", "        def yielder():\n", "            for i in range(0, random.randint(15, 20)):\n", "                time.sleep(0.1)\n", "                yield None\n", "        for img in progress.tqdm(yielder()):\n", "            pass\n", "        return \"done\"\n", "    load_random_btn.click(load_random, {text, textb}, text2)\n", "        \n", "    # manual progress\n", "    def clean_imgs(text, progress=gr.Progress()):\n", "        progress(0.2, desc=\"Collecting Images\")\n", "        time.sleep(1)\n", "        progress(0.5, desc=\"Cleaning Images\")\n", "        time.sleep(1.5)\n", "        progress(0.8, desc=\"Sending Images\")\n", "        time.sleep(1.5)\n", "        return \"done\"\n", "    clean_imgs_btn.click(clean_imgs, text, text2)\n", "\n", "    # no progress\n", "    def wait(text):\n", "        time.sleep(4)\n", "        return \"done\"\n", "    wait_btn.click(wait, text, text2)\n", "\n", "    # multiple progressions\n", "    def do_all(data, progress=gr.Progress()):\n", "        load_set(data[text], data[textb], progress)\n", "        load_random(data, progress)\n", "        clean_imgs(data[text], progress)\n", "        progress(None)\n", "        wait(text)\n", "        return \"done\"\n", "    do_all_btn.click(do_all, {text, textb}, text2)\n", "\n", "    def track_tqdm(data, progress=gr.Progress(track_tqdm=True)):\n", "        for i in tqdm.tqdm(range(5), desc=\"outer\"):\n", "            for j in tqdm.tqdm(range(4), desc=\"inner\"):\n", "                time.sleep(1)\n", "        return \"done\"\n", "    track_tqdm_btn.click(track_tqdm, {text, textb}, text2)\n", "\n", "    def bind_internal_tqdm(data, progress=gr.Progress(track_tqdm=True)):\n", "        outdir = \"__tmp/\" + str(uuid4())\n", "        load_dataset(\"beans\", split=\"train\", cache_dir=outdir)\n", "        shutil.rmtree(outdir)\n", "        return \"done\"\n", "    bind_internal_tqdm_btn.click(bind_internal_tqdm, {text, textb}, text2)\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.queue(concurrency_count=20).launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: progress"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio tqdm datasets"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import random\n", "import time\n", "import tqdm\n", "from datasets import load_dataset\n", "import shutil\n", "from uuid import uuid4\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        text = gr.Textbox()\n", "        textb = gr.Textbox()\n", "    with gr.Row():\n", "        load_set_btn = gr.Button(\"Load Set\")\n", "        load_nested_set_btn = gr.Button(\"Load Nested Set\")\n", "        load_random_btn = gr.Button(\"Load Random\")\n", "        clean_imgs_btn = gr.Button(\"Clean Images\")\n", "        wait_btn = gr.Button(\"Wait\")\n", "        do_all_btn = gr.Button(\"Do All\")\n", "        track_tqdm_btn = gr.Button(\"Bind TQDM\")\n", "        bind_internal_tqdm_btn = gr.Button(\"Bind Internal TQDM\")\n", "\n", "    text2 = gr.Textbox()\n", "\n", "    # track list\n", "    def load_set(text, text2, progress=gr.Progress()):\n", "        imgs = [None] * 24\n", "        for img in progress.tqdm(imgs, desc=\"Loading from list\"):\n", "            time.sleep(0.1)\n", "        return \"done\"\n", "    load_set_btn.click(load_set, [text, textb], text2)\n", "\n", "    # track nested list\n", "    def load_nested_set(text, text2, progress=gr.Progress()):\n", "        imgs = [[None] * 8] * 3\n", "        for img_set in progress.tqdm(imgs, desc=\"Nested list\"):\n", "            time.sleep(2)\n", "            for img in progress.tqdm(img_set, desc=\"inner list\"):\n", "                time.sleep(0.1)\n", "        return \"done\"\n", "    load_nested_set_btn.click(load_nested_set, [text, textb], text2)\n", "\n", "    # track iterable of unknown length\n", "    def load_random(data, progress=gr.Progress()):\n", "        def yielder():\n", "            for i in range(0, random.randint(15, 20)):\n", "                time.sleep(0.1)\n", "                yield None\n", "        for img in progress.tqdm(yielder()):\n", "            pass\n", "        return \"done\"\n", "    load_random_btn.click(load_random, {text, textb}, text2)\n", "        \n", "    # manual progress\n", "    def clean_imgs(text, progress=gr.Progress()):\n", "        progress(0.2, desc=\"Collecting Images\")\n", "        time.sleep(1)\n", "        progress(0.5, desc=\"Cleaning Images\")\n", "        time.sleep(1.5)\n", "        progress(0.8, desc=\"Sending Images\")\n", "        time.sleep(1.5)\n", "        return \"done\"\n", "    clean_imgs_btn.click(clean_imgs, text, text2)\n", "\n", "    # no progress\n", "    def wait(text):\n", "        time.sleep(4)\n", "        return \"done\"\n", "    wait_btn.click(wait, text, text2)\n", "\n", "    # multiple progressions\n", "    def do_all(data, progress=gr.Progress()):\n", "        load_set(data[text], data[textb], progress)\n", "        load_random(data, progress)\n", "        clean_imgs(data[text], progress)\n", "        progress(None)\n", "        wait(text)\n", "        return \"done\"\n", "    do_all_btn.click(do_all, {text, textb}, text2)\n", "\n", "    def track_tqdm(data, progress=gr.Progress(track_tqdm=True)):\n", "        for i in tqdm.tqdm(range(5), desc=\"outer\"):\n", "            for j in tqdm.tqdm(range(4), desc=\"inner\"):\n", "                time.sleep(1)\n", "        return \"done\"\n", "    track_tqdm_btn.click(track_tqdm, {text, textb}, text2)\n", "\n", "    def bind_internal_tqdm(data, progress=gr.Progress(track_tqdm=True)):\n", "        outdir = \"__tmp/\" + str(uuid4())\n", "        load_dataset(\"beans\", split=\"train\", cache_dir=outdir)\n", "        shutil.rmtree(outdir)\n", "        return \"done\"\n", "    bind_internal_tqdm_btn.click(bind_internal_tqdm, {text, textb}, text2)\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/progress/run.py

@@ -94,4 +94,4 @@ def bind_internal_tqdm(data, progress=gr.Progress(track_tqdm=True)):
 
 
 if __name__ == "__main__":
-    demo.queue(concurrency_count=20).launch()
+    demo.launch()