Add documentation around custom utterances

azimuth/modules/base_classes/dask_module.py

@@ -138,7 +138,9 @@ def start_task(self, client: Client, custom_query: Dict[str, Any]) -> "DaskModul
         log.info(f"Starting custom query {self.name}")
         # pure=false to be sure that everything is rerun.
         # Using self.name as key as we don't have indices
-        self.future = client.submit(self.compute, custom_query, key=self.name, pure=False)
+        self.future = client.submit(
+            self.compute, custom_query, key=f"{self.name}_{hash(str(custom_query))}", pure=False
+        )
         # Tell that this future is for custom use only.
         self.future.is_custom = True
         self.add_done_callback(self.on_end)

azimuth/routers/v1/custom_utterances.py

@@ -13,7 +13,7 @@
 from azimuth.config import AzimuthConfig
 from azimuth.modules.perturbation_testing import PerturbationTestingModule
 from azimuth.task_manager import TaskManager
-from azimuth.types import DatasetSplitName, SupportedMethod
+from azimuth.types import DatasetSplitName, ModuleOptions, SupportedMethod
 from azimuth.types.perturbation_testing import (
     PRETTY_PERTURBATION_TYPES,
     PerturbationTestFailureReason,
@@ -22,7 +22,7 @@
 )
 from azimuth.types.task import SaliencyResponse
 from azimuth.utils.conversion import orjson_dumps
-from azimuth.utils.routers import get_custom_task_result, require_available_model
+from azimuth.utils.routers import get_custom_task_result, require_pipeline_index
 
 router = APIRouter()
 
@@ -92,16 +92,17 @@ def get_perturbed_utterances(
     description="Get saliency for custom utterances.",
     tags=TAGS,
     response_model=List[SaliencyResponse],
-    dependencies=[Depends(require_available_model)],
 )
 def get_saliency(
     utterances: List[str] = Query([], title="Utterances"),
+    pipeline_index: int = Depends(require_pipeline_index),
     task_manager: TaskManager = Depends(get_task_manager),
 ) -> List[SaliencyResponse]:
     task_result: List[SaliencyResponse] = get_custom_task_result(
         SupportedMethod.Saliency,
         task_manager=task_manager,
         custom_query={task_manager.config.columns.text_input: utterances},
+        mod_options=ModuleOptions(pipeline_index=pipeline_index),
     )
 
     return task_result

azimuth/task_manager.py

@@ -203,8 +203,8 @@ def get_custom_task(
             else:
                 self.current_tasks[key] = task
 
-            if not task.done():
-                task.start_task(self.client, custom_query)
+            # Always start task
+            task.start_task(self.client, custom_query)
             return key, task
         else:
             log.warning("Task not found!", name=task_name)

docs/docs/user-guide/behavioral-testing-summary.md

@@ -35,3 +35,7 @@ A summary of the test results can be **downloaded**, as well as the **modified s
 both the evaluation set and training set. Download the required file by clicking **Export**
 :material-download: in the upper right corner of the table and then selecting the appropriate item
 from the menu.
+
+!!! tip "Augment custom utterance"
+
+    See our [:material-link: Custom Utterances page](custom-utterances.md) to learn how to augment additional data.

docs/docs/user-guide/custom-utterances.md

@@ -0,0 +1,64 @@
+# Custom utterances
+
+Azimuth has limited support for "custom utterances", i.e. utterances not part of the initial dataset.
+
+Azimuth offers support through API routes accessible locally at [`http://0.0.0.0:8091/docs#/Custom%20Utterances%20v1`](http://0.0.0.0:8091/docs#/Custom%20Utterances%20v1).
+
+## Data augmentation
+
+To augment a list of utterances, you can follow these intructions:
+
+=== "Python"
+
+    ```python
+    from pprint import pprint
+
+    import requests
+
+    utterances = ["welcome to Azimuth",
+                  "Don't forget to star our repo!"]
+
+    response = requests.get("http://0.0.0.0:8091/custom_utterances/perturbed_utterances",
+                            params={"utterances": utterances}).json()
+    pprint([r["perturbedUtterance"] for r in response])
+    ```
+
+=== "Output"
+
+    ```bash
+    >>> [ 'pls welcome to Azimuth',
+          'please welcome to Azimuth',
+          ...,
+          'Do not forget to star our repo!'
+          ]
+    ```
+
+## Saliency
+
+If a pipeline allows it, you can get saliency of a custom utterance by doing the following:
+
+=== "Python"
+
+    ```python
+    from pprint import pprint
+
+    import requests
+
+    utterances = ["welcome to Azimuth",
+                  "Don't forget to star our repo!"]
+
+    response = requests.get("http://0.0.0.0:8091/custom_utterances/saliency",
+                            params={"utterances": utterances, "pipelineIndex": 0}).json()
+    pprint(response)
+    ```
+
+=== "Output"
+
+    ```bash
+    >>> [{'saliency': [0.08587087690830231,
+                        ...
+                       ],
+          'tokens': ['[CLS]', 'welcome', 'to', 'az', '##im', '##uth', '[SEP]']},
+          ...
+         ]
+    ```

docs/mkdocs.yml

@@ -81,6 +81,7 @@ nav:
           - user-guide/exploration-space/utterances-table.md
           - user-guide/exploration-space/utterance-details.md
       - user-guide/settings.md
+      - user-guide/custom-utterances.md
   - Reference:
       - reference/index.md
       - Configuration: