Improve Text2SQL Metrics: Refactoring, New Execution Metric, and Bug Fixes

prepare/tasks/text2sql.py

@@ -23,3 +23,23 @@
     "tasks.text2sql",
     overwrite=True,
 )
+
+add_to_catalog(
+    Task(
+        input_fields={
+            "id": str,
+            "utterance": str,
+            "hint": Optional[str],
+            "db": SQLDatabase,
+        },
+        reference_fields={"query": str},
+        prediction_type=str,
+        metrics=[
+            "metrics.text2sql.execution_logic_accuracy",
+            "metrics.text2sql.non_execution_accuracy",
+            "metrics.anls",
+        ],
+    ),
+    "tasks.text2sql_logic",
+    overwrite=True,
+)

src/unitxt/catalog/metrics/text2sql/execution_logic_accuracy.json

@@ -0,0 +1,3 @@
+{
+    "__type__": "sql_execution_accuracy"
+}

src/unitxt/catalog/tasks/text2sql_logic.json

@@ -0,0 +1,18 @@
+{
+    "__type__": "task",
+    "input_fields": {
+        "id": "str",
+        "utterance": "str",
+        "hint": "Optional[str]",
+        "db": "SQLDatabase"
+    },
+    "reference_fields": {
+        "query": "str"
+    },
+    "prediction_type": "str",
+    "metrics": [
+        "metrics.text2sql.execution_logic_accuracy",
+        "metrics.text2sql.non_execution_accuracy",
+        "metrics.anls"
+    ]
+}

src/unitxt/dataset.py

@@ -53,7 +53,6 @@
 from .span_lableing_operators import __file__ as _
 from .split_utils import __file__ as _
 from .splitters import __file__ as _
-from .sql_utils import __file__ as _
 from .standard import __file__ as _
 from .stream import __file__ as _
 from .stream_operators import __file__ as _
@@ -62,6 +61,7 @@
 from .system_prompts import __file__ as _
 from .task import __file__ as _
 from .templates import __file__ as _
+from .text2sql_utils import __file__ as _
 from .text_utils import __file__ as _
 from .type_utils import __file__ as _
 from .types import __file__ as _

src/unitxt/loaders.py

@@ -1153,7 +1153,7 @@ class LoadFromAPI(Loader):
     chunksize: int = 100000
     loader_limit: Optional[int] = None
     streaming: bool = False
-    api_key_env_var: Optional[str] = ""
+    api_key_env_var: Optional[str] = None
     headers: Optional[Dict[str, Any]] = None
     data_field: str = "data"
     method: str = "GET"

src/unitxt/metric.py

@@ -51,7 +51,6 @@
 from .span_lableing_operators import __file__ as _
 from .split_utils import __file__ as _
 from .splitters import __file__ as _
-from .sql_utils import __file__ as _
 from .standard import __file__ as _
 from .stream import __file__ as _
 from .stream_operators import __file__ as _
@@ -60,6 +59,7 @@
 from .system_prompts import __file__ as _
 from .task import __file__ as _
 from .templates import __file__ as _
+from .text2sql_utils import __file__ as _
 from .text_utils import __file__ as _
 from .type_utils import __file__ as _
 from .types import __file__ as _