MAINT: Refactoring General Identifiers and ScorerIdentifier

doc/api.rst

@@ -258,6 +258,25 @@ API Reference
     XPIAProcessingCallback
     XPIAStatus
 
+:py:mod:`pyrit.identifiers`
+===========================
+
+.. automodule:: pyrit.identifiers
+    :no-members:
+    :no-inherited-members:
+
+.. autosummary::
+    :nosignatures:
+    :toctree: _autosummary/
+
+    class_name_to_snake_case
+    Identifiable
+    Identifier
+    IdentifierT
+    IdentifierType
+    ScorerIdentifier
+    snake_case_to_class_name
+
 :py:mod:`pyrit.memory`
 ======================
 
@@ -332,9 +351,6 @@ API Reference
     group_conversation_message_pieces_by_sequence
     group_message_pieces_into_conversations
     HarmDefinition
-    Identifiable
-    Identifier
-    IdentifierType
     ImagePathDataTypeSerializer
     AllowedCategories
     AttackOutcome
@@ -566,7 +582,6 @@ API Reference
     Scorer
     ScorerEvalDatasetFiles
     ScorerEvaluator
-    ScorerIdentifier
     ScorerMetrics
     ScorerMetricsWithIdentity
     ScorerPrinter

doc/code/scoring/8_scorer_metrics.ipynb

@@ -39,13 +39,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Found default environment files: ['C:\\\\Users\\\\songjustin\\\\.pyrit\\\\.env']\n",
-      "Loaded environment file: C:\\Users\\songjustin\\.pyrit\\.env\n",
+      "Found default environment files: ['C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\rlundeen\\\\.pyrit\\\\.env.local']\n",
+      "Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env\n",
+      "Loaded environment file: C:\\Users\\rlundeen\\.pyrit\\.env.local\n",
       "Scorer Identity:\n",
       "  Type: SelfAskRefusalScorer\n",
       "  System Prompt: You are an evaluator tasked with determining if th...\n",
-      "  Target Info: {'__type__': 'OpenAIChatTarget', 'model_name': 'gpt-4o', 'temperature': 0.9}\n",
-      "  Identity Hash: fad78ac914bc01f9e11edc7e25e04b41e201e1d5e0578ea5ea518096ff78eae5\n"
+      "  Target Info: {'__type__': 'OpenAIChatTarget', 'model_name': 'gpt-4o-japan-nilfilter', 'temperature': 0.9}\n",
+      "  Identity Hash: e848d270400aaabd673fd5413c7d08b7342a92fad461b48db2d681ab9d782b5e\n"
      ]
     }
    ],
@@ -60,14 +61,14 @@
     "refusal_scorer = SelfAskRefusalScorer(chat_target=OpenAIChatTarget(temperature=0.9))\n",
     "\n",
     "# View the scorer's full identity - this determines which metrics apply\n",
-    "scorer_identity = refusal_scorer.scorer_identifier\n",
+    "scorer_identity = refusal_scorer.get_identifier()\n",
     "print(\"Scorer Identity:\")\n",
-    "print(f\"  Type: {scorer_identity.type}\")\n",
+    "print(f\"  Type: {scorer_identity.class_name}\")\n",
     "print(\n",
     "    f\"  System Prompt: {scorer_identity.system_prompt_template[:50] if scorer_identity.system_prompt_template else 'None'}...\"\n",
     ")\n",
     "print(f\"  Target Info: {scorer_identity.target_info}\")\n",
-    "print(f\"  Identity Hash: {scorer_identity.compute_hash()}\")"
+    "print(f\"  Identity Hash: {scorer_identity.hash}\")"
    ]
   },
   {
@@ -145,14 +146,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Evaluation Metrics:\n",
-      "  Dataset Name: exploit.csv_hate_speech.csv_info_integrity.csv_privacy.csv_self_harm.csv_sexual.csv_violence.csv\n",
-      "  Dataset Version: 1.0_1.0_1.0_1.0_1.0_1.0_1.0\n",
-      "  F1 Score: 0.8536585365853658\n",
-      "  Accuracy: 0.8378378378378378\n",
-      "  Precision: 0.7777777777777778\n",
-      "  Recall: 0.9459459459459459\n",
-      "  Avg Score time: 0.556185774099258 seconds\n"
+      "No cached metrics found for this scorer configuration.\n"
      ]
     }
    ],
@@ -211,13 +205,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Evaluation Metrics:\n",
-      "  Dataset Name: exploit.csv\n",
-      "  Dataset Version: 1.0\n",
-      "  Mean Absolute Error: 0.348\n",
-      "  Krippendorff Alpha: 0.462\n",
-      "  P value: 0.0005\n",
-      "  Avg Score time: 0.6900209628577744 seconds\n"
+      "No cached metrics found for this scorer configuration.\n"
      ]
     }
    ],
@@ -669,7 +657,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.13.5"
   }
  },
  "nbformat": 4,

doc/code/scoring/8_scorer_metrics.py

@@ -5,7 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.2
+#       jupytext_version: 1.18.1
+#   kernelspec:
+#     display_name: pyrit (3.13.5)
+#     language: python
+#     name: python3
 # ---
 
 # %% [markdown]
@@ -43,14 +47,14 @@
 refusal_scorer = SelfAskRefusalScorer(chat_target=OpenAIChatTarget(temperature=0.9))
 
 # View the scorer's full identity - this determines which metrics apply
-scorer_identity = refusal_scorer.scorer_identifier
+scorer_identity = refusal_scorer.get_identifier()
 print("Scorer Identity:")
-print(f"  Type: {scorer_identity.type}")
+print(f"  Type: {scorer_identity.class_name}")
 print(
     f"  System Prompt: {scorer_identity.system_prompt_template[:50] if scorer_identity.system_prompt_template else 'None'}..."
 )
 print(f"  Target Info: {scorer_identity.target_info}")
-print(f"  Identity Hash: {scorer_identity.compute_hash()}")
+print(f"  Identity Hash: {scorer_identity.hash}")
 
 # %% [markdown]
 # ### Objective Metrics